2024-12-04 19:13:44,209 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-12-04 19:13:44,223 main DEBUG Took 0.011995 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-04 19:13:44,224 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-04 19:13:44,224 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-04 19:13:44,226 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-04 19:13:44,227 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 19:13:44,236 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-04 19:13:44,247 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 19:13:44,249 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 19:13:44,250 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 19:13:44,251 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 19:13:44,251 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 19:13:44,251 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 19:13:44,253 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 19:13:44,253 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 19:13:44,254 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 19:13:44,254 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 19:13:44,255 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 19:13:44,255 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 19:13:44,256 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 19:13:44,256 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 19:13:44,257 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 19:13:44,257 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 19:13:44,258 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 19:13:44,258 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 19:13:44,259 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 19:13:44,259 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 19:13:44,260 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 19:13:44,260 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 19:13:44,261 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 19:13:44,261 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 19:13:44,262 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 19:13:44,262 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-04 19:13:44,264 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 19:13:44,266 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-04 19:13:44,268 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-04 19:13:44,268 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-04 19:13:44,270 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-04 19:13:44,270 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-04 19:13:44,280 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-04 19:13:44,282 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-04 19:13:44,284 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-04 19:13:44,284 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-04 19:13:44,284 main DEBUG createAppenders(={Console}) 2024-12-04 19:13:44,285 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-12-04 19:13:44,286 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-12-04 19:13:44,286 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-12-04 19:13:44,286 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-04 19:13:44,287 main DEBUG OutputStream closed 2024-12-04 19:13:44,287 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-04 19:13:44,287 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-04 19:13:44,287 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-12-04 19:13:44,369 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-04 19:13:44,372 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-04 19:13:44,373 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-04 19:13:44,375 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-04 19:13:44,375 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-04 19:13:44,376 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-04 19:13:44,376 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-04 19:13:44,377 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-04 19:13:44,377 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-04 19:13:44,378 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-04 19:13:44,378 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-04 19:13:44,379 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-04 19:13:44,379 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-04 19:13:44,380 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-04 19:13:44,380 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-04 19:13:44,380 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-04 19:13:44,381 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-04 19:13:44,382 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-04 19:13:44,384 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-04 19:13:44,385 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-12-04 19:13:44,385 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-04 19:13:44,386 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-12-04T19:13:44,591 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1db4ba39-9940-6e0b-9624-ffcf7fe27afc 2024-12-04 19:13:44,594 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-04 19:13:44,594 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-04T19:13:44,602 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-12-04T19:13:44,620 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-04T19:13:44,634 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1db4ba39-9940-6e0b-9624-ffcf7fe27afc/cluster_45aec480-86fd-40c3-b6dd-2ba84309c8cd, deleteOnExit=true 2024-12-04T19:13:44,634 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-04T19:13:44,635 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1db4ba39-9940-6e0b-9624-ffcf7fe27afc/test.cache.data in system properties and HBase conf 2024-12-04T19:13:44,636 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1db4ba39-9940-6e0b-9624-ffcf7fe27afc/hadoop.tmp.dir in system properties and HBase conf 2024-12-04T19:13:44,636 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1db4ba39-9940-6e0b-9624-ffcf7fe27afc/hadoop.log.dir in system properties and HBase conf 2024-12-04T19:13:44,637 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1db4ba39-9940-6e0b-9624-ffcf7fe27afc/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-04T19:13:44,637 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1db4ba39-9940-6e0b-9624-ffcf7fe27afc/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-04T19:13:44,638 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-04T19:13:44,720 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-04T19:13:44,798 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-04T19:13:44,801 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1db4ba39-9940-6e0b-9624-ffcf7fe27afc/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-04T19:13:44,801 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1db4ba39-9940-6e0b-9624-ffcf7fe27afc/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-04T19:13:44,802 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1db4ba39-9940-6e0b-9624-ffcf7fe27afc/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-04T19:13:44,802 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1db4ba39-9940-6e0b-9624-ffcf7fe27afc/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-04T19:13:44,802 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1db4ba39-9940-6e0b-9624-ffcf7fe27afc/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-04T19:13:44,803 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1db4ba39-9940-6e0b-9624-ffcf7fe27afc/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-04T19:13:44,803 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1db4ba39-9940-6e0b-9624-ffcf7fe27afc/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-04T19:13:44,803 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1db4ba39-9940-6e0b-9624-ffcf7fe27afc/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-04T19:13:44,804 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1db4ba39-9940-6e0b-9624-ffcf7fe27afc/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-04T19:13:44,804 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1db4ba39-9940-6e0b-9624-ffcf7fe27afc/nfs.dump.dir in system properties and HBase conf 2024-12-04T19:13:44,804 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1db4ba39-9940-6e0b-9624-ffcf7fe27afc/java.io.tmpdir in system properties and HBase conf 2024-12-04T19:13:44,805 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1db4ba39-9940-6e0b-9624-ffcf7fe27afc/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-04T19:13:44,805 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1db4ba39-9940-6e0b-9624-ffcf7fe27afc/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-04T19:13:44,805 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1db4ba39-9940-6e0b-9624-ffcf7fe27afc/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-04T19:13:45,816 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-04T19:13:45,882 INFO [Time-limited test {}] log.Log(170): Logging initialized @2335ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-04T19:13:45,946 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T19:13:45,998 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T19:13:46,017 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T19:13:46,017 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T19:13:46,018 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-04T19:13:46,030 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T19:13:46,032 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@71c9bba7{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1db4ba39-9940-6e0b-9624-ffcf7fe27afc/hadoop.log.dir/,AVAILABLE} 2024-12-04T19:13:46,033 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@d942d39{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-04T19:13:46,238 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@322e478f{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1db4ba39-9940-6e0b-9624-ffcf7fe27afc/java.io.tmpdir/jetty-localhost-37517-hadoop-hdfs-3_4_1-tests_jar-_-any-714483685234520267/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-04T19:13:46,245 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4b008a6a{HTTP/1.1, (http/1.1)}{localhost:37517} 2024-12-04T19:13:46,245 INFO [Time-limited test {}] server.Server(415): Started @2698ms 2024-12-04T19:13:46,770 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T19:13:46,778 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T19:13:46,779 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T19:13:46,779 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T19:13:46,779 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-04T19:13:46,780 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3de8243e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1db4ba39-9940-6e0b-9624-ffcf7fe27afc/hadoop.log.dir/,AVAILABLE} 2024-12-04T19:13:46,781 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@74fde7a3{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-04T19:13:46,887 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@42a79998{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1db4ba39-9940-6e0b-9624-ffcf7fe27afc/java.io.tmpdir/jetty-localhost-38591-hadoop-hdfs-3_4_1-tests_jar-_-any-8843478346392913209/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T19:13:46,888 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4975be1c{HTTP/1.1, (http/1.1)}{localhost:38591} 2024-12-04T19:13:46,888 INFO [Time-limited test {}] server.Server(415): Started @3342ms 2024-12-04T19:13:46,938 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-04T19:13:47,926 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1db4ba39-9940-6e0b-9624-ffcf7fe27afc/cluster_45aec480-86fd-40c3-b6dd-2ba84309c8cd/data/data1/current/BP-523244764-172.17.0.2-1733339625272/current, will proceed with Du for space computation calculation, 2024-12-04T19:13:47,926 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1db4ba39-9940-6e0b-9624-ffcf7fe27afc/cluster_45aec480-86fd-40c3-b6dd-2ba84309c8cd/data/data2/current/BP-523244764-172.17.0.2-1733339625272/current, will proceed with Du for space computation calculation, 2024-12-04T19:13:47,949 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-04T19:13:47,997 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x7fbc01e933a47bf9 with lease ID 0x2663445acdb1fbde: Processing first storage report for DS-7f8b8dcc-4719-4b22-830c-c4c2b45edc59 from datanode DatanodeRegistration(127.0.0.1:34819, datanodeUuid=e1a5776f-89d1-4b59-8da9-1063d7e61e79, infoPort=38683, infoSecurePort=0, ipcPort=39455, storageInfo=lv=-57;cid=testClusterID;nsid=2058171377;c=1733339625272) 2024-12-04T19:13:47,998 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x7fbc01e933a47bf9 with lease ID 0x2663445acdb1fbde: from storage DS-7f8b8dcc-4719-4b22-830c-c4c2b45edc59 node DatanodeRegistration(127.0.0.1:34819, datanodeUuid=e1a5776f-89d1-4b59-8da9-1063d7e61e79, infoPort=38683, infoSecurePort=0, ipcPort=39455, storageInfo=lv=-57;cid=testClusterID;nsid=2058171377;c=1733339625272), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-04T19:13:47,999 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x7fbc01e933a47bf9 with lease ID 0x2663445acdb1fbde: Processing first storage report for DS-d00ad35c-bee3-4481-8d39-c0204ef82a29 from datanode DatanodeRegistration(127.0.0.1:34819, datanodeUuid=e1a5776f-89d1-4b59-8da9-1063d7e61e79, infoPort=38683, infoSecurePort=0, ipcPort=39455, storageInfo=lv=-57;cid=testClusterID;nsid=2058171377;c=1733339625272) 2024-12-04T19:13:47,999 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x7fbc01e933a47bf9 with lease ID 0x2663445acdb1fbde: from storage DS-d00ad35c-bee3-4481-8d39-c0204ef82a29 node DatanodeRegistration(127.0.0.1:34819, datanodeUuid=e1a5776f-89d1-4b59-8da9-1063d7e61e79, infoPort=38683, infoSecurePort=0, ipcPort=39455, storageInfo=lv=-57;cid=testClusterID;nsid=2058171377;c=1733339625272), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T19:13:48,016 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1db4ba39-9940-6e0b-9624-ffcf7fe27afc 2024-12-04T19:13:48,077 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1db4ba39-9940-6e0b-9624-ffcf7fe27afc/cluster_45aec480-86fd-40c3-b6dd-2ba84309c8cd/zookeeper_0, clientPort=60754, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1db4ba39-9940-6e0b-9624-ffcf7fe27afc/cluster_45aec480-86fd-40c3-b6dd-2ba84309c8cd/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1db4ba39-9940-6e0b-9624-ffcf7fe27afc/cluster_45aec480-86fd-40c3-b6dd-2ba84309c8cd/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-04T19:13:48,085 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=60754 2024-12-04T19:13:48,098 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T19:13:48,102 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T19:13:48,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741825_1001 (size=7) 2024-12-04T19:13:48,711 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d with version=8 2024-12-04T19:13:48,711 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/hbase-staging 2024-12-04T19:13:48,786 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-04T19:13:49,020 INFO [Time-limited test {}] client.ConnectionUtils(128): master/9ba2734cd39f:0 server-side Connection retries=45 2024-12-04T19:13:49,029 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T19:13:49,029 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-04T19:13:49,033 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-04T19:13:49,033 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T19:13:49,033 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-04T19:13:49,144 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-04T19:13:49,192 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-04T19:13:49,199 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-04T19:13:49,203 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-04T19:13:49,223 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 4090 (auto-detected) 2024-12-04T19:13:49,224 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-12-04T19:13:49,240 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:46667 2024-12-04T19:13:49,261 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:46667 connecting to ZooKeeper ensemble=127.0.0.1:60754 2024-12-04T19:13:49,526 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:466670x0, quorum=127.0.0.1:60754, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-04T19:13:49,529 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:46667-0x101a1c3bdc90000 connected 2024-12-04T19:13:49,681 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T19:13:49,686 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T19:13:49,697 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:46667-0x101a1c3bdc90000, quorum=127.0.0.1:60754, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T19:13:49,701 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d, hbase.cluster.distributed=false 2024-12-04T19:13:49,726 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:46667-0x101a1c3bdc90000, quorum=127.0.0.1:60754, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-04T19:13:49,731 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46667 2024-12-04T19:13:49,732 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46667 2024-12-04T19:13:49,732 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46667 2024-12-04T19:13:49,732 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46667 2024-12-04T19:13:49,734 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46667 2024-12-04T19:13:49,820 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/9ba2734cd39f:0 server-side Connection retries=45 2024-12-04T19:13:49,821 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T19:13:49,821 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-04T19:13:49,821 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-04T19:13:49,822 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T19:13:49,822 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-04T19:13:49,824 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-04T19:13:49,826 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-04T19:13:49,827 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33969 2024-12-04T19:13:49,828 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:33969 connecting to ZooKeeper ensemble=127.0.0.1:60754 2024-12-04T19:13:49,829 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T19:13:49,831 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T19:13:49,846 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:339690x0, quorum=127.0.0.1:60754, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-04T19:13:49,846 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33969-0x101a1c3bdc90001, quorum=127.0.0.1:60754, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T19:13:49,846 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:33969-0x101a1c3bdc90001 connected 2024-12-04T19:13:49,850 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-04T19:13:49,856 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-04T19:13:49,858 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33969-0x101a1c3bdc90001, quorum=127.0.0.1:60754, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-04T19:13:49,863 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33969-0x101a1c3bdc90001, quorum=127.0.0.1:60754, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-04T19:13:49,863 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33969 2024-12-04T19:13:49,864 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33969 2024-12-04T19:13:49,864 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33969 2024-12-04T19:13:49,864 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33969 2024-12-04T19:13:49,865 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33969 2024-12-04T19:13:49,879 DEBUG [M:0;9ba2734cd39f:46667 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;9ba2734cd39f:46667 2024-12-04T19:13:49,880 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/9ba2734cd39f,46667,1733339628877 2024-12-04T19:13:49,894 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33969-0x101a1c3bdc90001, quorum=127.0.0.1:60754, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T19:13:49,894 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46667-0x101a1c3bdc90000, quorum=127.0.0.1:60754, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T19:13:49,896 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:46667-0x101a1c3bdc90000, quorum=127.0.0.1:60754, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/9ba2734cd39f,46667,1733339628877 2024-12-04T19:13:49,930 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33969-0x101a1c3bdc90001, quorum=127.0.0.1:60754, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-04T19:13:49,930 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46667-0x101a1c3bdc90000, quorum=127.0.0.1:60754, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T19:13:49,930 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33969-0x101a1c3bdc90001, quorum=127.0.0.1:60754, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T19:13:49,931 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:46667-0x101a1c3bdc90000, quorum=127.0.0.1:60754, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-04T19:13:49,932 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/9ba2734cd39f,46667,1733339628877 from backup master directory 2024-12-04T19:13:49,940 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33969-0x101a1c3bdc90001, quorum=127.0.0.1:60754, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T19:13:49,940 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46667-0x101a1c3bdc90000, quorum=127.0.0.1:60754, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/9ba2734cd39f,46667,1733339628877 2024-12-04T19:13:49,940 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46667-0x101a1c3bdc90000, quorum=127.0.0.1:60754, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T19:13:49,941 WARN [master/9ba2734cd39f:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-04T19:13:49,941 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=9ba2734cd39f,46667,1733339628877 2024-12-04T19:13:49,943 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-04T19:13:49,945 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-04T19:13:49,996 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/hbase.id] with ID: 9baaea35-8dd3-4653-9b7c-03e5b792e644 2024-12-04T19:13:49,996 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/.tmp/hbase.id 2024-12-04T19:13:50,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741826_1002 (size=42) 2024-12-04T19:13:50,410 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/.tmp/hbase.id]:[hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/hbase.id] 2024-12-04T19:13:50,456 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T19:13:50,460 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-04T19:13:50,478 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 15ms. 2024-12-04T19:13:50,541 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33969-0x101a1c3bdc90001, quorum=127.0.0.1:60754, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T19:13:50,541 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46667-0x101a1c3bdc90000, quorum=127.0.0.1:60754, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T19:13:50,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741827_1003 (size=196) 2024-12-04T19:13:50,980 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T19:13:50,983 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-04T19:13:50,996 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:13:50,999 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T19:13:51,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741828_1004 (size=1189) 2024-12-04T19:13:51,447 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/MasterData/data/master/store 2024-12-04T19:13:51,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741829_1005 (size=34) 2024-12-04T19:13:51,469 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-04T19:13:51,472 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T19:13:51,473 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-04T19:13:51,473 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T19:13:51,474 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T19:13:51,476 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-04T19:13:51,476 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T19:13:51,476 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T19:13:51,477 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733339631473Disabling compacts and flushes for region at 1733339631473Disabling writes for close at 1733339631476 (+3 ms)Writing region close event to WAL at 1733339631476Closed at 1733339631476 2024-12-04T19:13:51,479 WARN [master/9ba2734cd39f:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/MasterData/data/master/store/.initializing 2024-12-04T19:13:51,479 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/MasterData/WALs/9ba2734cd39f,46667,1733339628877 2024-12-04T19:13:51,487 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-04T19:13:51,501 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9ba2734cd39f%2C46667%2C1733339628877, suffix=, logDir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/MasterData/WALs/9ba2734cd39f,46667,1733339628877, archiveDir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/MasterData/oldWALs, maxLogs=10 2024-12-04T19:13:51,521 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/MasterData/WALs/9ba2734cd39f,46667,1733339628877/9ba2734cd39f%2C46667%2C1733339628877.1733339631505, exclude list is [], retry=0 2024-12-04T19:13:51,538 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34819,DS-7f8b8dcc-4719-4b22-830c-c4c2b45edc59,DISK] 2024-12-04T19:13:51,541 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-04T19:13:51,571 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/MasterData/WALs/9ba2734cd39f,46667,1733339628877/9ba2734cd39f%2C46667%2C1733339628877.1733339631505 2024-12-04T19:13:51,572 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38683:38683)] 2024-12-04T19:13:51,572 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-04T19:13:51,573 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T19:13:51,575 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T19:13:51,576 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T19:13:51,607 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T19:13:51,628 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-04T19:13:51,631 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:13:51,633 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T19:13:51,633 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T19:13:51,636 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-04T19:13:51,637 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:13:51,638 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:13:51,638 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T19:13:51,640 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-04T19:13:51,641 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:13:51,642 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:13:51,642 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T19:13:51,644 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-04T19:13:51,644 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:13:51,645 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:13:51,646 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T19:13:51,649 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-04T19:13:51,650 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-04T19:13:51,655 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T19:13:51,656 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T19:13:51,659 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-04T19:13:51,663 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T19:13:51,668 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T19:13:51,669 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64180193, jitterRate=-0.04364059865474701}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-04T19:13:51,673 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733339631587Initializing all the Stores at 1733339631589 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733339631589Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733339631590 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733339631590Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733339631590Cleaning up temporary data from old regions at 1733339631656 (+66 ms)Region opened successfully at 1733339631673 (+17 ms) 2024-12-04T19:13:51,675 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-04T19:13:51,704 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@601677f4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=9ba2734cd39f/172.17.0.2:0 2024-12-04T19:13:51,732 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-04T19:13:51,741 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-04T19:13:51,741 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-04T19:13:51,744 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-04T19:13:51,745 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-04T19:13:51,749 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-12-04T19:13:51,749 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-04T19:13:51,771 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-04T19:13:51,780 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46667-0x101a1c3bdc90000, quorum=127.0.0.1:60754, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-04T19:13:51,866 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-04T19:13:51,870 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-04T19:13:51,872 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46667-0x101a1c3bdc90000, quorum=127.0.0.1:60754, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-04T19:13:51,883 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-04T19:13:51,885 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-04T19:13:51,889 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46667-0x101a1c3bdc90000, quorum=127.0.0.1:60754, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-04T19:13:51,898 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-04T19:13:51,899 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46667-0x101a1c3bdc90000, quorum=127.0.0.1:60754, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-04T19:13:51,908 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-04T19:13:51,928 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46667-0x101a1c3bdc90000, quorum=127.0.0.1:60754, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-04T19:13:51,940 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-04T19:13:51,950 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46667-0x101a1c3bdc90000, quorum=127.0.0.1:60754, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-04T19:13:51,950 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33969-0x101a1c3bdc90001, quorum=127.0.0.1:60754, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-04T19:13:51,951 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46667-0x101a1c3bdc90000, quorum=127.0.0.1:60754, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T19:13:51,951 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33969-0x101a1c3bdc90001, quorum=127.0.0.1:60754, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T19:13:51,954 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=9ba2734cd39f,46667,1733339628877, sessionid=0x101a1c3bdc90000, setting cluster-up flag (Was=false) 2024-12-04T19:13:51,988 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46667-0x101a1c3bdc90000, quorum=127.0.0.1:60754, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T19:13:51,988 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33969-0x101a1c3bdc90001, quorum=127.0.0.1:60754, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T19:13:52,025 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-04T19:13:52,031 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=9ba2734cd39f,46667,1733339628877 2024-12-04T19:13:52,056 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46667-0x101a1c3bdc90000, quorum=127.0.0.1:60754, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T19:13:52,056 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33969-0x101a1c3bdc90001, quorum=127.0.0.1:60754, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T19:13:52,088 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-04T19:13:52,092 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=9ba2734cd39f,46667,1733339628877 2024-12-04T19:13:52,103 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-04T19:13:52,169 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-04T19:13:52,171 INFO [RS:0;9ba2734cd39f:33969 {}] regionserver.HRegionServer(746): ClusterId : 9baaea35-8dd3-4653-9b7c-03e5b792e644 2024-12-04T19:13:52,173 DEBUG [RS:0;9ba2734cd39f:33969 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-04T19:13:52,180 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-04T19:13:52,184 DEBUG [RS:0;9ba2734cd39f:33969 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-04T19:13:52,184 DEBUG [RS:0;9ba2734cd39f:33969 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-04T19:13:52,186 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-04T19:13:52,190 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 9ba2734cd39f,46667,1733339628877 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-04T19:13:52,194 DEBUG [RS:0;9ba2734cd39f:33969 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-04T19:13:52,194 DEBUG [RS:0;9ba2734cd39f:33969 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@306fec73, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=9ba2734cd39f/172.17.0.2:0 2024-12-04T19:13:52,198 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/9ba2734cd39f:0, corePoolSize=5, maxPoolSize=5 2024-12-04T19:13:52,198 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/9ba2734cd39f:0, corePoolSize=5, maxPoolSize=5 2024-12-04T19:13:52,198 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/9ba2734cd39f:0, corePoolSize=5, maxPoolSize=5 2024-12-04T19:13:52,198 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/9ba2734cd39f:0, corePoolSize=5, maxPoolSize=5 2024-12-04T19:13:52,198 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/9ba2734cd39f:0, corePoolSize=10, maxPoolSize=10 2024-12-04T19:13:52,198 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/9ba2734cd39f:0, corePoolSize=1, maxPoolSize=1 2024-12-04T19:13:52,199 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/9ba2734cd39f:0, corePoolSize=2, maxPoolSize=2 2024-12-04T19:13:52,199 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/9ba2734cd39f:0, corePoolSize=1, maxPoolSize=1 2024-12-04T19:13:52,201 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733339662201 2024-12-04T19:13:52,203 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-04T19:13:52,205 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-04T19:13:52,206 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-04T19:13:52,206 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-04T19:13:52,209 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-04T19:13:52,209 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-04T19:13:52,210 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-04T19:13:52,210 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-04T19:13:52,211 DEBUG [RS:0;9ba2734cd39f:33969 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;9ba2734cd39f:33969 2024-12-04T19:13:52,211 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:13:52,212 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-04T19:13:52,211 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-04T19:13:52,216 INFO [RS:0;9ba2734cd39f:33969 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-04T19:13:52,216 INFO [RS:0;9ba2734cd39f:33969 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-04T19:13:52,216 DEBUG [RS:0;9ba2734cd39f:33969 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-04T19:13:52,217 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-04T19:13:52,218 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-04T19:13:52,218 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-04T19:13:52,219 INFO [RS:0;9ba2734cd39f:33969 {}] regionserver.HRegionServer(2659): reportForDuty to master=9ba2734cd39f,46667,1733339628877 with port=33969, startcode=1733339629791 2024-12-04T19:13:52,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741831_1007 (size=1321) 2024-12-04T19:13:52,224 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-04T19:13:52,225 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-04T19:13:52,228 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/9ba2734cd39f:0:becomeActiveMaster-HFileCleaner.large.0-1733339632226,5,FailOnTimeoutGroup] 2024-12-04T19:13:52,229 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/9ba2734cd39f:0:becomeActiveMaster-HFileCleaner.small.0-1733339632228,5,FailOnTimeoutGroup] 2024-12-04T19:13:52,229 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-04T19:13:52,229 DEBUG [RS:0;9ba2734cd39f:33969 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-04T19:13:52,229 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-04T19:13:52,230 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-04T19:13:52,230 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-04T19:13:52,258 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49315, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-04T19:13:52,265 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46667 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 9ba2734cd39f,33969,1733339629791 2024-12-04T19:13:52,268 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46667 {}] master.ServerManager(517): Registering regionserver=9ba2734cd39f,33969,1733339629791 2024-12-04T19:13:52,283 DEBUG [RS:0;9ba2734cd39f:33969 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d 2024-12-04T19:13:52,283 DEBUG [RS:0;9ba2734cd39f:33969 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:44307 2024-12-04T19:13:52,284 DEBUG [RS:0;9ba2734cd39f:33969 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-04T19:13:52,298 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46667-0x101a1c3bdc90000, quorum=127.0.0.1:60754, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-04T19:13:52,299 DEBUG [RS:0;9ba2734cd39f:33969 {}] zookeeper.ZKUtil(111): regionserver:33969-0x101a1c3bdc90001, quorum=127.0.0.1:60754, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/9ba2734cd39f,33969,1733339629791 2024-12-04T19:13:52,299 WARN [RS:0;9ba2734cd39f:33969 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-04T19:13:52,299 INFO [RS:0;9ba2734cd39f:33969 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T19:13:52,299 DEBUG [RS:0;9ba2734cd39f:33969 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/WALs/9ba2734cd39f,33969,1733339629791 2024-12-04T19:13:52,301 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [9ba2734cd39f,33969,1733339629791] 2024-12-04T19:13:52,321 INFO [RS:0;9ba2734cd39f:33969 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-04T19:13:52,333 INFO [RS:0;9ba2734cd39f:33969 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-04T19:13:52,337 INFO [RS:0;9ba2734cd39f:33969 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-04T19:13:52,337 INFO [RS:0;9ba2734cd39f:33969 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T19:13:52,338 INFO [RS:0;9ba2734cd39f:33969 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-04T19:13:52,342 INFO [RS:0;9ba2734cd39f:33969 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-04T19:13:52,343 INFO [RS:0;9ba2734cd39f:33969 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-04T19:13:52,344 DEBUG [RS:0;9ba2734cd39f:33969 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/9ba2734cd39f:0, corePoolSize=1, maxPoolSize=1 2024-12-04T19:13:52,344 DEBUG [RS:0;9ba2734cd39f:33969 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/9ba2734cd39f:0, corePoolSize=1, maxPoolSize=1 2024-12-04T19:13:52,344 DEBUG [RS:0;9ba2734cd39f:33969 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/9ba2734cd39f:0, corePoolSize=1, maxPoolSize=1 2024-12-04T19:13:52,344 DEBUG [RS:0;9ba2734cd39f:33969 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/9ba2734cd39f:0, corePoolSize=1, maxPoolSize=1 2024-12-04T19:13:52,344 DEBUG [RS:0;9ba2734cd39f:33969 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/9ba2734cd39f:0, corePoolSize=1, maxPoolSize=1 2024-12-04T19:13:52,344 DEBUG [RS:0;9ba2734cd39f:33969 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/9ba2734cd39f:0, corePoolSize=2, maxPoolSize=2 2024-12-04T19:13:52,344 DEBUG [RS:0;9ba2734cd39f:33969 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/9ba2734cd39f:0, corePoolSize=1, maxPoolSize=1 2024-12-04T19:13:52,344 DEBUG [RS:0;9ba2734cd39f:33969 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/9ba2734cd39f:0, corePoolSize=1, maxPoolSize=1 2024-12-04T19:13:52,345 DEBUG [RS:0;9ba2734cd39f:33969 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/9ba2734cd39f:0, corePoolSize=1, maxPoolSize=1 2024-12-04T19:13:52,345 DEBUG [RS:0;9ba2734cd39f:33969 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/9ba2734cd39f:0, corePoolSize=1, maxPoolSize=1 2024-12-04T19:13:52,345 DEBUG [RS:0;9ba2734cd39f:33969 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/9ba2734cd39f:0, corePoolSize=1, maxPoolSize=1 2024-12-04T19:13:52,345 DEBUG [RS:0;9ba2734cd39f:33969 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/9ba2734cd39f:0, corePoolSize=1, maxPoolSize=1 2024-12-04T19:13:52,345 DEBUG [RS:0;9ba2734cd39f:33969 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/9ba2734cd39f:0, corePoolSize=3, maxPoolSize=3 2024-12-04T19:13:52,345 DEBUG [RS:0;9ba2734cd39f:33969 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0, corePoolSize=3, maxPoolSize=3 2024-12-04T19:13:52,346 INFO [RS:0;9ba2734cd39f:33969 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-04T19:13:52,346 INFO [RS:0;9ba2734cd39f:33969 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-04T19:13:52,346 INFO [RS:0;9ba2734cd39f:33969 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T19:13:52,346 INFO [RS:0;9ba2734cd39f:33969 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-04T19:13:52,346 INFO [RS:0;9ba2734cd39f:33969 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-04T19:13:52,347 INFO [RS:0;9ba2734cd39f:33969 {}] hbase.ChoreService(168): Chore ScheduledChore name=9ba2734cd39f,33969,1733339629791-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-04T19:13:52,363 INFO [RS:0;9ba2734cd39f:33969 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-04T19:13:52,364 INFO [RS:0;9ba2734cd39f:33969 {}] hbase.ChoreService(168): Chore ScheduledChore name=9ba2734cd39f,33969,1733339629791-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T19:13:52,364 INFO [RS:0;9ba2734cd39f:33969 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T19:13:52,364 INFO [RS:0;9ba2734cd39f:33969 {}] regionserver.Replication(171): 9ba2734cd39f,33969,1733339629791 started 2024-12-04T19:13:52,379 INFO [RS:0;9ba2734cd39f:33969 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T19:13:52,380 INFO [RS:0;9ba2734cd39f:33969 {}] regionserver.HRegionServer(1482): Serving as 9ba2734cd39f,33969,1733339629791, RpcServer on 9ba2734cd39f/172.17.0.2:33969, sessionid=0x101a1c3bdc90001 2024-12-04T19:13:52,380 DEBUG [RS:0;9ba2734cd39f:33969 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-04T19:13:52,380 DEBUG [RS:0;9ba2734cd39f:33969 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 9ba2734cd39f,33969,1733339629791 2024-12-04T19:13:52,381 DEBUG [RS:0;9ba2734cd39f:33969 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9ba2734cd39f,33969,1733339629791' 2024-12-04T19:13:52,381 DEBUG [RS:0;9ba2734cd39f:33969 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-04T19:13:52,382 DEBUG [RS:0;9ba2734cd39f:33969 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-04T19:13:52,382 DEBUG [RS:0;9ba2734cd39f:33969 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-04T19:13:52,382 DEBUG [RS:0;9ba2734cd39f:33969 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-04T19:13:52,383 DEBUG [RS:0;9ba2734cd39f:33969 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 9ba2734cd39f,33969,1733339629791 2024-12-04T19:13:52,383 DEBUG [RS:0;9ba2734cd39f:33969 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9ba2734cd39f,33969,1733339629791' 2024-12-04T19:13:52,383 DEBUG [RS:0;9ba2734cd39f:33969 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-04T19:13:52,383 DEBUG [RS:0;9ba2734cd39f:33969 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-04T19:13:52,384 DEBUG [RS:0;9ba2734cd39f:33969 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-04T19:13:52,384 INFO [RS:0;9ba2734cd39f:33969 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-04T19:13:52,384 INFO [RS:0;9ba2734cd39f:33969 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-04T19:13:52,495 INFO [RS:0;9ba2734cd39f:33969 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-04T19:13:52,498 INFO [RS:0;9ba2734cd39f:33969 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9ba2734cd39f%2C33969%2C1733339629791, suffix=, logDir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/WALs/9ba2734cd39f,33969,1733339629791, archiveDir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/oldWALs, maxLogs=32 2024-12-04T19:13:52,513 DEBUG [RS:0;9ba2734cd39f:33969 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/WALs/9ba2734cd39f,33969,1733339629791/9ba2734cd39f%2C33969%2C1733339629791.1733339632500, exclude list is [], retry=0 2024-12-04T19:13:52,517 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34819,DS-7f8b8dcc-4719-4b22-830c-c4c2b45edc59,DISK] 2024-12-04T19:13:52,520 INFO [RS:0;9ba2734cd39f:33969 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/WALs/9ba2734cd39f,33969,1733339629791/9ba2734cd39f%2C33969%2C1733339629791.1733339632500 2024-12-04T19:13:52,521 DEBUG [RS:0;9ba2734cd39f:33969 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38683:38683)] 2024-12-04T19:13:52,626 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-04T19:13:52,627 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d 2024-12-04T19:13:52,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741833_1009 (size=32) 2024-12-04T19:13:53,046 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T19:13:53,051 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-04T19:13:53,054 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-04T19:13:53,055 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:13:53,056 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T19:13:53,056 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-04T19:13:53,060 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-04T19:13:53,060 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:13:53,062 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T19:13:53,062 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-04T19:13:53,064 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-04T19:13:53,064 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:13:53,065 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T19:13:53,066 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-04T19:13:53,068 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-04T19:13:53,068 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:13:53,069 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T19:13:53,069 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-04T19:13:53,071 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/hbase/meta/1588230740 2024-12-04T19:13:53,072 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/hbase/meta/1588230740 2024-12-04T19:13:53,074 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-04T19:13:53,075 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-04T19:13:53,076 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-04T19:13:53,078 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-04T19:13:53,081 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T19:13:53,082 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59361315, jitterRate=-0.11544747650623322}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-04T19:13:53,084 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733339633047Initializing all the Stores at 1733339633050 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733339633050Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733339633050Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733339633050Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733339633050Cleaning up temporary data from old regions at 1733339633075 (+25 ms)Region opened successfully at 1733339633084 (+9 ms) 2024-12-04T19:13:53,084 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-04T19:13:53,084 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-04T19:13:53,084 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-04T19:13:53,084 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-04T19:13:53,084 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-04T19:13:53,086 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-04T19:13:53,086 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733339633084Disabling compacts and flushes for region at 1733339633084Disabling writes for close at 1733339633084Writing region close event to WAL at 1733339633085 (+1 ms)Closed at 1733339633086 (+1 ms) 2024-12-04T19:13:53,089 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-04T19:13:53,089 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-04T19:13:53,094 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-04T19:13:53,101 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-04T19:13:53,104 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-04T19:13:53,261 DEBUG [9ba2734cd39f:46667 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-04T19:13:53,273 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=9ba2734cd39f,33969,1733339629791 2024-12-04T19:13:53,278 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9ba2734cd39f,33969,1733339629791, state=OPENING 2024-12-04T19:13:53,351 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-04T19:13:53,361 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46667-0x101a1c3bdc90000, quorum=127.0.0.1:60754, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T19:13:53,361 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33969-0x101a1c3bdc90001, quorum=127.0.0.1:60754, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T19:13:53,363 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T19:13:53,363 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T19:13:53,366 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-04T19:13:53,369 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=9ba2734cd39f,33969,1733339629791}] 2024-12-04T19:13:53,553 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-04T19:13:53,556 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60483, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-04T19:13:53,570 INFO [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-04T19:13:53,570 INFO [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T19:13:53,571 INFO [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-04T19:13:53,574 INFO [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9ba2734cd39f%2C33969%2C1733339629791.meta, suffix=.meta, logDir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/WALs/9ba2734cd39f,33969,1733339629791, archiveDir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/oldWALs, maxLogs=32 2024-12-04T19:13:53,589 DEBUG [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/WALs/9ba2734cd39f,33969,1733339629791/9ba2734cd39f%2C33969%2C1733339629791.meta.1733339633576.meta, exclude list is [], retry=0 2024-12-04T19:13:53,593 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34819,DS-7f8b8dcc-4719-4b22-830c-c4c2b45edc59,DISK] 2024-12-04T19:13:53,596 INFO [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/WALs/9ba2734cd39f,33969,1733339629791/9ba2734cd39f%2C33969%2C1733339629791.meta.1733339633576.meta 2024-12-04T19:13:53,596 DEBUG [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38683:38683)] 2024-12-04T19:13:53,597 DEBUG [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-04T19:13:53,598 DEBUG [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-04T19:13:53,601 DEBUG [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-04T19:13:53,605 INFO [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-04T19:13:53,609 DEBUG [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-04T19:13:53,609 DEBUG [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T19:13:53,609 DEBUG [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-04T19:13:53,609 DEBUG [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-04T19:13:53,612 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-04T19:13:53,614 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-04T19:13:53,614 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:13:53,615 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T19:13:53,615 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-04T19:13:53,617 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-04T19:13:53,617 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:13:53,618 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T19:13:53,618 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-04T19:13:53,619 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-04T19:13:53,619 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:13:53,620 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T19:13:53,620 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-04T19:13:53,622 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-04T19:13:53,622 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:13:53,622 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T19:13:53,623 DEBUG [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-04T19:13:53,624 DEBUG [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/hbase/meta/1588230740 2024-12-04T19:13:53,627 DEBUG [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/hbase/meta/1588230740 2024-12-04T19:13:53,629 DEBUG [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-04T19:13:53,630 DEBUG [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-04T19:13:53,631 DEBUG [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-04T19:13:53,634 DEBUG [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-04T19:13:53,636 INFO [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72797395, jitterRate=0.0847657173871994}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-04T19:13:53,637 DEBUG [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-04T19:13:53,638 DEBUG [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733339633610Writing region info on filesystem at 1733339633610Initializing all the Stores at 1733339633612 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733339633612Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733339633612Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733339633612Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733339633612Cleaning up temporary data from old regions at 1733339633630 (+18 ms)Running coprocessor post-open hooks at 1733339633637 (+7 ms)Region opened successfully at 1733339633638 (+1 ms) 2024-12-04T19:13:53,644 INFO [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733339633543 2024-12-04T19:13:53,654 DEBUG [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-04T19:13:53,654 INFO [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-04T19:13:53,656 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=9ba2734cd39f,33969,1733339629791 2024-12-04T19:13:53,658 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9ba2734cd39f,33969,1733339629791, state=OPEN 2024-12-04T19:13:53,787 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33969-0x101a1c3bdc90001, quorum=127.0.0.1:60754, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-04T19:13:53,787 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46667-0x101a1c3bdc90000, quorum=127.0.0.1:60754, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-04T19:13:53,788 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T19:13:53,788 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T19:13:53,789 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:13:53,799 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-04T19:13:53,799 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9ba2734cd39f,33969,1733339629791 in 420 msec 2024-12-04T19:13:53,806 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-04T19:13:53,807 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 707 msec 2024-12-04T19:13:53,808 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-04T19:13:53,808 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-04T19:13:53,825 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T19:13:53,826 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ba2734cd39f,33969,1733339629791, seqNum=-1] 2024-12-04T19:13:53,843 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T19:13:53,845 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39067, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T19:13:53,865 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.7330 sec 2024-12-04T19:13:53,865 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733339633865, completionTime=-1 2024-12-04T19:13:53,868 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-12-04T19:13:53,868 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-04T19:13:53,891 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-12-04T19:13:53,891 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733339693891 2024-12-04T19:13:53,891 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733339753891 2024-12-04T19:13:53,891 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 23 msec 2024-12-04T19:13:53,893 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9ba2734cd39f,46667,1733339628877-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T19:13:53,893 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9ba2734cd39f,46667,1733339628877-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T19:13:53,893 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9ba2734cd39f,46667,1733339628877-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T19:13:53,895 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-9ba2734cd39f:46667, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T19:13:53,895 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-04T19:13:53,896 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-04T19:13:53,901 DEBUG [master/9ba2734cd39f:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-04T19:13:53,922 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.981sec 2024-12-04T19:13:53,923 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-04T19:13:53,925 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-04T19:13:53,926 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-04T19:13:53,926 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-04T19:13:53,927 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-04T19:13:53,927 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9ba2734cd39f,46667,1733339628877-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-04T19:13:53,928 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9ba2734cd39f,46667,1733339628877-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-04T19:13:53,936 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-04T19:13:53,937 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-04T19:13:53,937 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9ba2734cd39f,46667,1733339628877-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T19:13:53,984 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@519673c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:13:53,986 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-04T19:13:53,986 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-04T19:13:53,989 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ba2734cd39f,46667,-1 for getting cluster id 2024-12-04T19:13:53,991 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T19:13:53,997 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9baaea35-8dd3-4653-9b7c-03e5b792e644' 2024-12-04T19:13:53,999 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T19:13:53,999 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9baaea35-8dd3-4653-9b7c-03e5b792e644" 2024-12-04T19:13:54,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@424b8b64, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:13:54,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ba2734cd39f,46667,-1] 2024-12-04T19:13:54,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T19:13:54,005 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:13:54,006 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56876, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T19:13:54,008 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b2d93e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:13:54,009 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T19:13:54,014 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ba2734cd39f,33969,1733339629791, seqNum=-1] 2024-12-04T19:13:54,015 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T19:13:54,017 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57566, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T19:13:54,037 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=9ba2734cd39f,46667,1733339628877 2024-12-04T19:13:54,048 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=272, ProcessCount=11, AvailableMemoryMB=15782 2024-12-04T19:13:54,056 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-04T19:13:54,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 9ba2734cd39f,46667,1733339628877 2024-12-04T19:13:54,079 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@27423f7d 2024-12-04T19:13:54,080 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-04T19:13:54,083 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56886, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-04T19:13:54,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-04T19:13:54,090 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T19:13:54,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-04T19:13:54,105 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-04T19:13:54,108 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-12-04T19:13:54,108 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:13:54,110 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-04T19:13:54,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-04T19:13:54,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741835_1011 (size=1061) 2024-12-04T19:13:54,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-04T19:13:54,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-04T19:13:54,536 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d 2024-12-04T19:13:54,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741836_1012 (size=53) 2024-12-04T19:13:54,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-04T19:13:54,948 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T19:13:54,948 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 97532d2585300ca10ffc4f6d5dda82c5, disabling compactions & flushes 2024-12-04T19:13:54,948 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:13:54,949 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:13:54,949 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. after waiting 0 ms 2024-12-04T19:13:54,949 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:13:54,949 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:13:54,950 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 97532d2585300ca10ffc4f6d5dda82c5: Waiting for close lock at 1733339634948Disabling compacts and flushes for region at 1733339634948Disabling writes for close at 1733339634949 (+1 ms)Writing region close event to WAL at 1733339634949Closed at 1733339634949 2024-12-04T19:13:54,954 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-04T19:13:54,958 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733339634954"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733339634954"}]},"ts":"1733339634954"} 2024-12-04T19:13:54,963 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-04T19:13:54,965 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-04T19:13:54,967 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733339634965"}]},"ts":"1733339634965"} 2024-12-04T19:13:54,971 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-04T19:13:54,972 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=97532d2585300ca10ffc4f6d5dda82c5, ASSIGN}] 2024-12-04T19:13:54,975 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=97532d2585300ca10ffc4f6d5dda82c5, ASSIGN 2024-12-04T19:13:54,976 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=97532d2585300ca10ffc4f6d5dda82c5, ASSIGN; state=OFFLINE, location=9ba2734cd39f,33969,1733339629791; forceNewPlan=false, retain=false 2024-12-04T19:13:55,129 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=97532d2585300ca10ffc4f6d5dda82c5, regionState=OPENING, regionLocation=9ba2734cd39f,33969,1733339629791 2024-12-04T19:13:55,136 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=97532d2585300ca10ffc4f6d5dda82c5, ASSIGN because future has completed 2024-12-04T19:13:55,137 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791}] 2024-12-04T19:13:55,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-04T19:13:55,299 INFO [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:13:55,299 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} 2024-12-04T19:13:55,300 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 97532d2585300ca10ffc4f6d5dda82c5 2024-12-04T19:13:55,300 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T19:13:55,300 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 97532d2585300ca10ffc4f6d5dda82c5 2024-12-04T19:13:55,300 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 97532d2585300ca10ffc4f6d5dda82c5 2024-12-04T19:13:55,303 INFO [StoreOpener-97532d2585300ca10ffc4f6d5dda82c5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 97532d2585300ca10ffc4f6d5dda82c5 2024-12-04T19:13:55,307 INFO [StoreOpener-97532d2585300ca10ffc4f6d5dda82c5-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-04T19:13:55,308 INFO [StoreOpener-97532d2585300ca10ffc4f6d5dda82c5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 97532d2585300ca10ffc4f6d5dda82c5 columnFamilyName A 2024-12-04T19:13:55,308 DEBUG [StoreOpener-97532d2585300ca10ffc4f6d5dda82c5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:13:55,309 INFO [StoreOpener-97532d2585300ca10ffc4f6d5dda82c5-1 {}] regionserver.HStore(327): Store=97532d2585300ca10ffc4f6d5dda82c5/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:13:55,310 INFO [StoreOpener-97532d2585300ca10ffc4f6d5dda82c5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 97532d2585300ca10ffc4f6d5dda82c5 2024-12-04T19:13:55,312 INFO [StoreOpener-97532d2585300ca10ffc4f6d5dda82c5-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-04T19:13:55,313 INFO [StoreOpener-97532d2585300ca10ffc4f6d5dda82c5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 97532d2585300ca10ffc4f6d5dda82c5 columnFamilyName B 2024-12-04T19:13:55,313 DEBUG [StoreOpener-97532d2585300ca10ffc4f6d5dda82c5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:13:55,314 INFO [StoreOpener-97532d2585300ca10ffc4f6d5dda82c5-1 {}] regionserver.HStore(327): Store=97532d2585300ca10ffc4f6d5dda82c5/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:13:55,314 INFO [StoreOpener-97532d2585300ca10ffc4f6d5dda82c5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 97532d2585300ca10ffc4f6d5dda82c5 2024-12-04T19:13:55,317 INFO [StoreOpener-97532d2585300ca10ffc4f6d5dda82c5-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-04T19:13:55,317 INFO [StoreOpener-97532d2585300ca10ffc4f6d5dda82c5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 97532d2585300ca10ffc4f6d5dda82c5 columnFamilyName C 2024-12-04T19:13:55,317 DEBUG [StoreOpener-97532d2585300ca10ffc4f6d5dda82c5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:13:55,318 INFO [StoreOpener-97532d2585300ca10ffc4f6d5dda82c5-1 {}] regionserver.HStore(327): Store=97532d2585300ca10ffc4f6d5dda82c5/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:13:55,319 INFO [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:13:55,319 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 97532d2585300ca10ffc4f6d5dda82c5 2024-12-04T19:13:55,320 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5 2024-12-04T19:13:55,321 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5 2024-12-04T19:13:55,323 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 97532d2585300ca10ffc4f6d5dda82c5 2024-12-04T19:13:55,323 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 97532d2585300ca10ffc4f6d5dda82c5 2024-12-04T19:13:55,324 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-04T19:13:55,327 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 97532d2585300ca10ffc4f6d5dda82c5 2024-12-04T19:13:55,330 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T19:13:55,331 INFO [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 97532d2585300ca10ffc4f6d5dda82c5; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75017893, jitterRate=0.11785371601581573}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-04T19:13:55,331 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 97532d2585300ca10ffc4f6d5dda82c5 2024-12-04T19:13:55,332 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 97532d2585300ca10ffc4f6d5dda82c5: Running coprocessor pre-open hook at 1733339635300Writing region info on filesystem at 1733339635300Initializing all the Stores at 1733339635302 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733339635302Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733339635303 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733339635303Cleaning up temporary data from old regions at 1733339635323 (+20 ms)Running coprocessor post-open hooks at 1733339635332 (+9 ms)Region opened successfully at 1733339635332 2024-12-04T19:13:55,334 INFO [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., pid=6, masterSystemTime=1733339635292 2024-12-04T19:13:55,338 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:13:55,338 INFO [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:13:55,339 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=97532d2585300ca10ffc4f6d5dda82c5, regionState=OPEN, openSeqNum=2, regionLocation=9ba2734cd39f,33969,1733339629791 2024-12-04T19:13:55,342 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 because future has completed 2024-12-04T19:13:55,348 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-04T19:13:55,348 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 in 207 msec 2024-12-04T19:13:55,352 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-04T19:13:55,352 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=97532d2585300ca10ffc4f6d5dda82c5, ASSIGN in 376 msec 2024-12-04T19:13:55,353 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-04T19:13:55,353 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733339635353"}]},"ts":"1733339635353"} 2024-12-04T19:13:55,356 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-04T19:13:55,358 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-04T19:13:55,362 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2650 sec 2024-12-04T19:13:56,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-04T19:13:56,276 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-04T19:13:56,285 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f5ede62, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:13:56,285 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ba2734cd39f,46667,-1 for getting cluster id 2024-12-04T19:13:56,285 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T19:13:56,287 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9baaea35-8dd3-4653-9b7c-03e5b792e644' 2024-12-04T19:13:56,288 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T19:13:56,288 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9baaea35-8dd3-4653-9b7c-03e5b792e644" 2024-12-04T19:13:56,288 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46cd3e42, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:13:56,288 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ba2734cd39f,46667,-1] 2024-12-04T19:13:56,288 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T19:13:56,289 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:13:56,290 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56918, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T19:13:56,292 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32c82fed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:13:56,293 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T19:13:56,295 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ba2734cd39f,33969,1733339629791, seqNum=-1] 2024-12-04T19:13:56,295 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T19:13:56,297 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57582, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T19:13:56,300 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-04T19:13:56,301 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 9ba2734cd39f,46667,1733339628877 2024-12-04T19:13:56,301 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@da7d497 2024-12-04T19:13:56,304 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-04T19:13:56,306 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56920, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-04T19:13:56,310 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3812e936, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:13:56,310 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ba2734cd39f,46667,-1 for getting cluster id 2024-12-04T19:13:56,310 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T19:13:56,312 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9baaea35-8dd3-4653-9b7c-03e5b792e644' 2024-12-04T19:13:56,312 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T19:13:56,312 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9baaea35-8dd3-4653-9b7c-03e5b792e644" 2024-12-04T19:13:56,313 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@483a22c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:13:56,313 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ba2734cd39f,46667,-1] 2024-12-04T19:13:56,313 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T19:13:56,313 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:13:56,315 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56938, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T19:13:56,316 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@749fb54b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:13:56,317 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e500c50, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:13:56,317 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ba2734cd39f,46667,-1 for getting cluster id 2024-12-04T19:13:56,317 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T19:13:56,319 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9baaea35-8dd3-4653-9b7c-03e5b792e644' 2024-12-04T19:13:56,319 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T19:13:56,319 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9baaea35-8dd3-4653-9b7c-03e5b792e644" 2024-12-04T19:13:56,320 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68cf4e74, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:13:56,320 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ba2734cd39f,46667,-1] 2024-12-04T19:13:56,320 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T19:13:56,321 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:13:56,322 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56954, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T19:13:56,323 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cbf9665, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:13:56,324 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68c12611, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:13:56,324 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ba2734cd39f,46667,-1 for getting cluster id 2024-12-04T19:13:56,324 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T19:13:56,326 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9baaea35-8dd3-4653-9b7c-03e5b792e644' 2024-12-04T19:13:56,326 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T19:13:56,326 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9baaea35-8dd3-4653-9b7c-03e5b792e644" 2024-12-04T19:13:56,327 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33bdb6a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:13:56,327 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ba2734cd39f,46667,-1] 2024-12-04T19:13:56,327 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T19:13:56,327 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:13:56,329 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56966, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T19:13:56,330 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27108bea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:13:56,331 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77155d64, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:13:56,331 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ba2734cd39f,46667,-1 for getting cluster id 2024-12-04T19:13:56,331 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T19:13:56,335 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9baaea35-8dd3-4653-9b7c-03e5b792e644' 2024-12-04T19:13:56,335 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T19:13:56,335 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9baaea35-8dd3-4653-9b7c-03e5b792e644" 2024-12-04T19:13:56,336 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b0e6697, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:13:56,336 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ba2734cd39f,46667,-1] 2024-12-04T19:13:56,336 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T19:13:56,339 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:13:56,340 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56984, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T19:13:56,342 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f7ca9c6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:13:56,343 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bdceae5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:13:56,343 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ba2734cd39f,46667,-1 for getting cluster id 2024-12-04T19:13:56,343 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T19:13:56,344 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9baaea35-8dd3-4653-9b7c-03e5b792e644' 2024-12-04T19:13:56,344 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T19:13:56,345 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9baaea35-8dd3-4653-9b7c-03e5b792e644" 2024-12-04T19:13:56,345 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45b21b95, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:13:56,345 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ba2734cd39f,46667,-1] 2024-12-04T19:13:56,345 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T19:13:56,346 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:13:56,347 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57002, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T19:13:56,348 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a8d6bd7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:13:56,349 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58df653d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:13:56,349 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ba2734cd39f,46667,-1 for getting cluster id 2024-12-04T19:13:56,349 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T19:13:56,350 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9baaea35-8dd3-4653-9b7c-03e5b792e644' 2024-12-04T19:13:56,351 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T19:13:56,351 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9baaea35-8dd3-4653-9b7c-03e5b792e644" 2024-12-04T19:13:56,351 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@bc03052, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:13:56,352 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ba2734cd39f,46667,-1] 2024-12-04T19:13:56,352 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T19:13:56,352 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:13:56,353 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57024, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T19:13:56,354 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@267419b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:13:56,355 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@85187e0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:13:56,355 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ba2734cd39f,46667,-1 for getting cluster id 2024-12-04T19:13:56,355 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T19:13:56,356 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9baaea35-8dd3-4653-9b7c-03e5b792e644' 2024-12-04T19:13:56,357 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T19:13:56,357 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9baaea35-8dd3-4653-9b7c-03e5b792e644" 2024-12-04T19:13:56,357 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d111af0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:13:56,357 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ba2734cd39f,46667,-1] 2024-12-04T19:13:56,357 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T19:13:56,358 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:13:56,359 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57034, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T19:13:56,359 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e9639cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:13:56,361 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1dc6fa21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:13:56,361 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ba2734cd39f,46667,-1 for getting cluster id 2024-12-04T19:13:56,361 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T19:13:56,362 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9baaea35-8dd3-4653-9b7c-03e5b792e644' 2024-12-04T19:13:56,362 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T19:13:56,362 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9baaea35-8dd3-4653-9b7c-03e5b792e644" 2024-12-04T19:13:56,362 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e406351, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:13:56,362 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ba2734cd39f,46667,-1] 2024-12-04T19:13:56,363 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T19:13:56,363 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:13:56,364 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57056, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T19:13:56,365 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27d22a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:13:56,366 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a990c61, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:13:56,366 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ba2734cd39f,46667,-1 for getting cluster id 2024-12-04T19:13:56,366 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T19:13:56,367 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9baaea35-8dd3-4653-9b7c-03e5b792e644' 2024-12-04T19:13:56,367 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T19:13:56,367 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9baaea35-8dd3-4653-9b7c-03e5b792e644" 2024-12-04T19:13:56,367 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c994daa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:13:56,368 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ba2734cd39f,46667,-1] 2024-12-04T19:13:56,368 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T19:13:56,368 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:13:56,369 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57082, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T19:13:56,370 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2171b18c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:13:56,375 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T19:13:56,375 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T19:13:56,375 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T19:13:56,376 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T19:13:56,376 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T19:13:56,376 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T19:13:56,377 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T19:13:56,377 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T19:13:56,378 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T19:13:56,378 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ba2734cd39f,33969,1733339629791, seqNum=-1] 2024-12-04T19:13:56,379 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T19:13:56,379 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ba2734cd39f,33969,1733339629791, seqNum=-1] 2024-12-04T19:13:56,379 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ba2734cd39f,33969,1733339629791, seqNum=-1] 2024-12-04T19:13:56,379 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T19:13:56,380 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T19:13:56,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ba2734cd39f,33969,1733339629791, seqNum=-1] 2024-12-04T19:13:56,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T19:13:56,381 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ba2734cd39f,33969,1733339629791, seqNum=-1] 2024-12-04T19:13:56,381 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57588, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T19:13:56,381 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T19:13:56,382 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57600, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T19:13:56,382 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ba2734cd39f,33969,1733339629791, seqNum=-1] 2024-12-04T19:13:56,382 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T19:13:56,382 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T19:13:56,383 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ba2734cd39f,33969,1733339629791, seqNum=-1] 2024-12-04T19:13:56,383 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57604, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T19:13:56,383 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T19:13:56,384 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ba2734cd39f,33969,1733339629791, seqNum=-1] 2024-12-04T19:13:56,384 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T19:13:56,384 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57610, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T19:13:56,385 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57624, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T19:13:56,386 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57644, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T19:13:56,386 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57632, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T19:13:56,386 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ba2734cd39f,33969,1733339629791, seqNum=-1] 2024-12-04T19:13:56,386 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T19:13:56,388 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57646, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T19:13:56,388 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57662, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T19:13:56,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-12-04T19:13:56,392 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T19:13:56,392 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2] 2024-12-04T19:13:56,395 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2] 2024-12-04T19:13:56,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-04T19:13:56,395 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2] 2024-12-04T19:13:56,396 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2] 2024-12-04T19:13:56,397 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T19:13:56,398 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2] 2024-12-04T19:13:56,398 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2] 2024-12-04T19:13:56,398 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2] 2024-12-04T19:13:56,401 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T19:13:56,404 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2] 2024-12-04T19:13:56,404 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2] 2024-12-04T19:13:56,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-04T19:13:56,531 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 97532d2585300ca10ffc4f6d5dda82c5 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-04T19:13:56,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(8855): Flush requested on 97532d2585300ca10ffc4f6d5dda82c5 2024-12-04T19:13:56,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97532d2585300ca10ffc4f6d5dda82c5, store=A 2024-12-04T19:13:56,552 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:13:56,552 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97532d2585300ca10ffc4f6d5dda82c5, store=B 2024-12-04T19:13:56,552 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:13:56,553 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97532d2585300ca10ffc4f6d5dda82c5, store=C 2024-12-04T19:13:56,553 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:13:56,598 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-04T19:13:56,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:13:56,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. as already flushing 2024-12-04T19:13:56,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:13:56,623 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:13:56,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:13:56,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:13:56,661 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/A/e6ff7769f71744978011ed7aa2a84e79 is 50, key is test_row_0/A:col10/1733339636527/Put/seqid=0 2024-12-04T19:13:56,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741837_1013 (size=14341) 2024-12-04T19:13:56,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-04T19:13:56,715 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/A/e6ff7769f71744978011ed7aa2a84e79 2024-12-04T19:13:56,765 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:13:56,765 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:13:56,765 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:13:56,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57588 deadline: 1733339696750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:13:56,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57662 deadline: 1733339696746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:13:56,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57600 deadline: 1733339696754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:13:56,773 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:13:56,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57632 deadline: 1733339696758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:13:56,781 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:13:56,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57646 deadline: 1733339696764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:13:56,793 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-04T19:13:56,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:56,807 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:56,807 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:56,808 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:56,808 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:56,808 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:56,808 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:13:56,808 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:13:56,808 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:56,809 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:56,809 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:13:56,809 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:56,809 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:56,809 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:13:56,809 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:13:56,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:13:56,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. as already flushing 2024-12-04T19:13:56,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:13:56,821 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:13:56,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:13:56,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:13:56,833 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/B/6756372be3a4446aa9c78e6d12de05f5 is 50, key is test_row_0/B:col10/1733339636527/Put/seqid=0 2024-12-04T19:13:56,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741838_1014 (size=12001) 2024-12-04T19:13:56,858 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/B/6756372be3a4446aa9c78e6d12de05f5 2024-12-04T19:13:56,910 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/C/fb6347ed02a843fabb95ecbad719fb4f is 50, key is test_row_0/C:col10/1733339636527/Put/seqid=0 2024-12-04T19:13:56,914 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:13:56,914 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:13:56,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57646 deadline: 1733339696913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:13:56,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57600 deadline: 1733339696912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:13:56,915 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:56,915 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:56,916 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:13:56,917 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:56,917 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:56,917 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:13:56,917 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:13:56,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57662 deadline: 1733339696914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:13:56,919 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:56,919 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:56,919 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:13:56,919 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:13:56,919 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:13:56,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57632 deadline: 1733339696914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:13:56,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57588 deadline: 1733339696914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:13:56,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:56,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:56,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:13:56,921 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:56,921 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:56,921 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:13:56,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741839_1015 (size=12001) 2024-12-04T19:13:56,941 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/C/fb6347ed02a843fabb95ecbad719fb4f 2024-12-04T19:13:56,964 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/A/e6ff7769f71744978011ed7aa2a84e79 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/e6ff7769f71744978011ed7aa2a84e79 2024-12-04T19:13:56,978 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-04T19:13:56,981 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/e6ff7769f71744978011ed7aa2a84e79, entries=200, sequenceid=13, filesize=14.0 K 2024-12-04T19:13:56,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:13:56,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. as already flushing 2024-12-04T19:13:56,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:13:56,985 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:13:56,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:13:56,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:13:56,990 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/B/6756372be3a4446aa9c78e6d12de05f5 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/6756372be3a4446aa9c78e6d12de05f5 2024-12-04T19:13:57,004 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/6756372be3a4446aa9c78e6d12de05f5, entries=150, sequenceid=13, filesize=11.7 K 2024-12-04T19:13:57,006 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/C/fb6347ed02a843fabb95ecbad719fb4f as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/fb6347ed02a843fabb95ecbad719fb4f 2024-12-04T19:13:57,022 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/fb6347ed02a843fabb95ecbad719fb4f, entries=150, sequenceid=13, filesize=11.7 K 2024-12-04T19:13:57,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-04T19:13:57,033 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 97532d2585300ca10ffc4f6d5dda82c5 in 499ms, sequenceid=13, compaction requested=false 2024-12-04T19:13:57,037 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-04T19:13:57,039 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 97532d2585300ca10ffc4f6d5dda82c5: 2024-12-04T19:13:57,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(8855): Flush requested on 97532d2585300ca10ffc4f6d5dda82c5 2024-12-04T19:13:57,135 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 97532d2585300ca10ffc4f6d5dda82c5 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-04T19:13:57,137 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733339636741, max timestamp=1733339637126Num uniques 859; 2024-12-04T19:13:57,138 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733339636741, max timestamp=1733339637126Num uniques 859; 2024-12-04T19:13:57,139 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-04T19:13:57,139 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T19:13:57,143 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-04T19:13:57,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97532d2585300ca10ffc4f6d5dda82c5, store=A 2024-12-04T19:13:57,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:13:57,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. as already flushing 2024-12-04T19:13:57,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:13:57,145 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:13:57,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:13:57,146 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T19:13:57,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97532d2585300ca10ffc4f6d5dda82c5, store=B 2024-12-04T19:13:57,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T19:13:57,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97532d2585300ca10ffc4f6d5dda82c5, store=C 2024-12-04T19:13:57,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T19:13:57,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:13:57,164 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/A/0da65db6ed684622a05bda6392f565b5 is 50, key is test_row_0/A:col10/1733339637129/Put/seqid=0 2024-12-04T19:13:57,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741840_1016 (size=12001) 2024-12-04T19:13:57,186 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/A/0da65db6ed684622a05bda6392f565b5 2024-12-04T19:13:57,215 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/B/954d40e092ec4b1aa44f5760c7adf224 is 50, key is test_row_0/B:col10/1733339637129/Put/seqid=0 2024-12-04T19:13:57,222 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:13:57,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57632 deadline: 1733339697205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:13:57,224 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:57,224 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:57,224 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:13:57,226 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:13:57,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57600 deadline: 1733339697218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:13:57,228 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:57,227 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:13:57,228 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:57,228 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:13:57,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57646 deadline: 1733339697218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:13:57,229 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:57,229 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:57,229 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:13:57,231 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:13:57,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57662 deadline: 1733339697223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:13:57,232 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:13:57,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57588 deadline: 1733339697224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:13:57,233 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:57,233 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:57,234 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:13:57,234 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:57,234 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:57,234 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:13:57,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741841_1017 (size=12001) 2024-12-04T19:13:57,304 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-04T19:13:57,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:13:57,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. as already flushing 2024-12-04T19:13:57,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:13:57,305 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:13:57,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:13:57,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:13:57,334 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:13:57,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57632 deadline: 1733339697333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:13:57,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:57,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:13:57,337 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:57,337 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:13:57,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57646 deadline: 1733339697334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:13:57,338 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:57,338 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:57,338 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:13:57,336 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:13:57,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57600 deadline: 1733339697334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:13:57,341 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:57,341 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:57,341 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:13:57,351 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:13:57,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57662 deadline: 1733339697342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:13:57,353 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:57,353 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:57,353 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:13:57,354 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:13:57,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57588 deadline: 1733339697342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:13:57,355 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:57,355 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:57,356 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:13:57,463 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-04T19:13:57,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:13:57,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. as already flushing 2024-12-04T19:13:57,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:13:57,464 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:13:57,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:13:57,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:13:57,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-04T19:13:57,544 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:13:57,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57632 deadline: 1733339697543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:13:57,546 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:57,546 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:57,546 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:13:57,548 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:13:57,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57646 deadline: 1733339697544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:13:57,549 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:57,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:57,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:13:57,555 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:13:57,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57600 deadline: 1733339697553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:13:57,558 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:57,558 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:57,558 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:13:57,565 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:13:57,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57662 deadline: 1733339697563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:13:57,566 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:57,567 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:57,567 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:13:57,567 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:13:57,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57588 deadline: 1733339697563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:13:57,568 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:57,568 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:57,568 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:13:57,619 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-04T19:13:57,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:13:57,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. as already flushing 2024-12-04T19:13:57,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:13:57,619 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:13:57,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:13:57,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:13:57,658 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.02 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/B/954d40e092ec4b1aa44f5760c7adf224 2024-12-04T19:13:57,683 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/C/7c9469d1b25141b3a123563a45af1c09 is 50, key is test_row_0/C:col10/1733339637129/Put/seqid=0 2024-12-04T19:13:57,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741842_1018 (size=12001) 2024-12-04T19:13:57,743 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/C/7c9469d1b25141b3a123563a45af1c09 2024-12-04T19:13:57,760 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/A/0da65db6ed684622a05bda6392f565b5 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/0da65db6ed684622a05bda6392f565b5 2024-12-04T19:13:57,775 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-04T19:13:57,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:13:57,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. as already flushing 2024-12-04T19:13:57,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:13:57,776 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:13:57,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:13:57,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:13:57,780 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/0da65db6ed684622a05bda6392f565b5, entries=150, sequenceid=36, filesize=11.7 K 2024-12-04T19:13:57,783 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/B/954d40e092ec4b1aa44f5760c7adf224 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/954d40e092ec4b1aa44f5760c7adf224 2024-12-04T19:13:57,797 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/954d40e092ec4b1aa44f5760c7adf224, entries=150, sequenceid=36, filesize=11.7 K 2024-12-04T19:13:57,803 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/C/7c9469d1b25141b3a123563a45af1c09 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/7c9469d1b25141b3a123563a45af1c09 2024-12-04T19:13:57,820 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/7c9469d1b25141b3a123563a45af1c09, entries=150, sequenceid=36, filesize=11.7 K 2024-12-04T19:13:57,822 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~71.17 KB/72875, heapSize ~203.05 KB/207928, currentSize=114.05 KB/116790 for 97532d2585300ca10ffc4f6d5dda82c5 in 687ms, sequenceid=36, compaction requested=false 2024-12-04T19:13:57,822 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 97532d2585300ca10ffc4f6d5dda82c5: 2024-12-04T19:13:57,866 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733339637190, max timestamp=1733339637854Num uniques 859; 2024-12-04T19:13:57,866 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T19:13:57,870 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733339637190, max timestamp=1733339637854Num uniques 859; 2024-12-04T19:13:57,870 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T19:13:57,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(8855): Flush requested on 97532d2585300ca10ffc4f6d5dda82c5 2024-12-04T19:13:57,874 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 97532d2585300ca10ffc4f6d5dda82c5 3/3 column families, dataSize=58.14 KB heapSize=164.17 KB 2024-12-04T19:13:57,875 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97532d2585300ca10ffc4f6d5dda82c5, store=A 2024-12-04T19:13:57,875 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T19:13:57,875 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97532d2585300ca10ffc4f6d5dda82c5, store=B 2024-12-04T19:13:57,875 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T19:13:57,875 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97532d2585300ca10ffc4f6d5dda82c5, store=C 2024-12-04T19:13:57,875 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733339637190, max timestamp=1733339637854Num uniques 859; 2024-12-04T19:13:57,875 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T19:13:57,875 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T19:13:57,894 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/A/0c45af55b8624ba189b927011fc9ce88 is 50, key is test_row_0/A:col10/1733339637194/Put/seqid=0 2024-12-04T19:13:57,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741843_1019 (size=14341) 2024-12-04T19:13:57,923 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=58 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/A/0c45af55b8624ba189b927011fc9ce88 2024-12-04T19:13:57,931 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733339637911, max timestamp=1733339637919Num uniques 859; 2024-12-04T19:13:57,931 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733339637911, max timestamp=1733339637924Num uniques 859; 2024-12-04T19:13:57,931 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T19:13:57,931 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T19:13:57,933 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-04T19:13:57,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:13:57,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. as already flushing 2024-12-04T19:13:57,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:13:57,933 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:13:57,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:13:57,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:13:57,940 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733339637911, max timestamp=1733339637919Num uniques 859; 2024-12-04T19:13:57,940 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-04T19:13:57,951 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/B/cf37ea9a01f44d7f8a51863f27ae80f6 is 50, key is test_row_0/B:col10/1733339637194/Put/seqid=0 2024-12-04T19:13:57,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741844_1020 (size=12001) 2024-12-04T19:13:57,968 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=58 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/B/cf37ea9a01f44d7f8a51863f27ae80f6 2024-12-04T19:13:57,990 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/C/4f39b560c9fc4f1e8f366336a570451c is 50, key is test_row_0/C:col10/1733339637194/Put/seqid=0 2024-12-04T19:13:57,991 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733339637979, max timestamp=1733339637986Num uniques 1018; 2024-12-04T19:13:57,991 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733339637979, max timestamp=1733339637986Num uniques 1009; 2024-12-04T19:13:57,991 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-04T19:13:57,991 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46213, new segment data size=9978, suffix heap size=132904,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=218 2024-12-04T19:13:57,999 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733339637979, max timestamp=1733339637986Num uniques 1018; 2024-12-04T19:13:57,999 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=6870, suffix heap size=133840,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=150 2024-12-04T19:13:58,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741845_1021 (size=12001) 2024-12-04T19:13:58,076 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733339638056, max timestamp=1733339638071Num uniques 1077; 2024-12-04T19:13:58,076 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-12-04T19:13:58,078 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733339638056, max timestamp=1733339638071Num uniques 1009; 2024-12-04T19:13:58,078 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733339638056, max timestamp=1733339638071Num uniques 1077; 2024-12-04T19:13:58,079 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=8102, suffix heap size=132904,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=177 2024-12-04T19:13:58,079 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-12-04T19:13:58,091 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-04T19:13:58,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:13:58,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. as already flushing 2024-12-04T19:13:58,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:13:58,092 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:13:58,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:13:58,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:13:58,123 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1733339638107, max timestamp=1733339638117Num uniques 1036; 2024-12-04T19:13:58,123 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733339638107, max timestamp=1733339638117Num uniques 1036; 2024-12-04T19:13:58,123 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=10806, suffix heap size=135712,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=236 2024-12-04T19:13:58,123 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=6870, suffix heap size=135712,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=150 2024-12-04T19:13:58,123 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733339638107, max timestamp=1733339638117Num uniques 1036; 2024-12-04T19:13:58,123 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=6870, suffix heap size=135712,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=150 2024-12-04T19:13:58,181 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1733339638153, max timestamp=1733339638176Num uniques 1095; 2024-12-04T19:13:58,181 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=13510, suffix heap size=141848,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=295 2024-12-04T19:13:58,181 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1733339638153, max timestamp=1733339638176Num uniques 1009; 2024-12-04T19:13:58,182 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=13510, suffix heap size=132904,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=295 2024-12-04T19:13:58,184 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13500, totalHeapSize=30920, min timestamp=1733339638153, max timestamp=1733339638177Num uniques 1009; 2024-12-04T19:13:58,185 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46204, new segment data size=13500, suffix heap size=132904,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=295 2024-12-04T19:13:58,225 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:13:58,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57600 deadline: 1733339698213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:13:58,226 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:13:58,226 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:58,226 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:58,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57662 deadline: 1733339698213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:13:58,226 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:13:58,227 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:58,227 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:58,227 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:13:58,227 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:13:58,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57588 deadline: 1733339698217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:13:58,228 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:58,228 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:58,228 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:13:58,228 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:13:58,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57646 deadline: 1733339698217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:13:58,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:13:58,229 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:58,229 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:58,229 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:13:58,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57632 deadline: 1733339698220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:13:58,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:58,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:58,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:13:58,250 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-04T19:13:58,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:13:58,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. as already flushing 2024-12-04T19:13:58,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:13:58,251 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:13:58,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:13:58,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:13:58,317 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-04T19:13:58,334 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:13:58,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57600 deadline: 1733339698333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:13:58,335 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:13:58,335 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:58,335 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:58,335 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:13:58,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57646 deadline: 1733339698334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:13:58,336 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:58,336 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:58,336 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:13:58,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:13:58,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57662 deadline: 1733339698334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:13:58,336 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:13:58,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57632 deadline: 1733339698334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:13:58,337 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:58,337 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:58,337 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:13:58,337 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:13:58,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57588 deadline: 1733339698334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:13:58,337 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:58,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:58,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:13:58,340 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:58,340 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:58,340 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:13:58,407 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-04T19:13:58,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:13:58,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. as already flushing 2024-12-04T19:13:58,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:13:58,407 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:13:58,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:13:58,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:13:58,410 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=58 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/C/4f39b560c9fc4f1e8f366336a570451c 2024-12-04T19:13:58,424 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/A/0c45af55b8624ba189b927011fc9ce88 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/0c45af55b8624ba189b927011fc9ce88 2024-12-04T19:13:58,442 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/0c45af55b8624ba189b927011fc9ce88, entries=200, sequenceid=58, filesize=14.0 K 2024-12-04T19:13:58,447 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/B/cf37ea9a01f44d7f8a51863f27ae80f6 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/cf37ea9a01f44d7f8a51863f27ae80f6 2024-12-04T19:13:58,475 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/cf37ea9a01f44d7f8a51863f27ae80f6, entries=150, sequenceid=58, filesize=11.7 K 2024-12-04T19:13:58,480 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/C/4f39b560c9fc4f1e8f366336a570451c as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/4f39b560c9fc4f1e8f366336a570451c 2024-12-04T19:13:58,505 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/4f39b560c9fc4f1e8f366336a570451c, entries=150, sequenceid=58, filesize=11.7 K 2024-12-04T19:13:58,507 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~86.11 KB/88176, currentSize=154.31 KB/158010 for 97532d2585300ca10ffc4f6d5dda82c5 in 634ms, sequenceid=58, compaction requested=true 2024-12-04T19:13:58,507 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 97532d2585300ca10ffc4f6d5dda82c5: 2024-12-04T19:13:58,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 97532d2585300ca10ffc4f6d5dda82c5:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T19:13:58,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:13:58,517 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 97532d2585300ca10ffc4f6d5dda82c5:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T19:13:58,518 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:13:58,518 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 97532d2585300ca10ffc4f6d5dda82c5:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T19:13:58,518 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T19:13:58,518 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T19:13:58,518 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T19:13:58,522 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T19:13:58,527 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1541): 97532d2585300ca10ffc4f6d5dda82c5/B is initiating minor compaction (all files) 2024-12-04T19:13:58,528 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 97532d2585300ca10ffc4f6d5dda82c5/B in TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:13:58,528 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/6756372be3a4446aa9c78e6d12de05f5, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/954d40e092ec4b1aa44f5760c7adf224, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/cf37ea9a01f44d7f8a51863f27ae80f6] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp, totalSize=35.2 K 2024-12-04T19:13:58,532 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 6756372be3a4446aa9c78e6d12de05f5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733339636408 2024-12-04T19:13:58,533 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 954d40e092ec4b1aa44f5760c7adf224, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1733339636741 2024-12-04T19:13:58,534 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting cf37ea9a01f44d7f8a51863f27ae80f6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1733339637194 2024-12-04T19:13:58,543 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40683 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T19:13:58,544 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1541): 97532d2585300ca10ffc4f6d5dda82c5/A is initiating minor compaction (all files) 2024-12-04T19:13:58,544 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 97532d2585300ca10ffc4f6d5dda82c5/A in TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:13:58,545 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/e6ff7769f71744978011ed7aa2a84e79, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/0da65db6ed684622a05bda6392f565b5, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/0c45af55b8624ba189b927011fc9ce88] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp, totalSize=39.7 K 2024-12-04T19:13:58,546 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting e6ff7769f71744978011ed7aa2a84e79, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733339636408 2024-12-04T19:13:58,547 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0da65db6ed684622a05bda6392f565b5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1733339636741 2024-12-04T19:13:58,548 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0c45af55b8624ba189b927011fc9ce88, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1733339637194 2024-12-04T19:13:58,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(8855): Flush requested on 97532d2585300ca10ffc4f6d5dda82c5 2024-12-04T19:13:58,548 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1733339638204, max timestamp=1733339638544Num uniques 1154; 2024-12-04T19:13:58,549 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=52850, new segment data size=7050, suffix heap size=147984,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1154, new segment cells count=154 2024-12-04T19:13:58,549 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1733339638204, max timestamp=1733339638544Num uniques 1154; 2024-12-04T19:13:58,549 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=52850, new segment data size=7050, suffix heap size=147984,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1154, new segment cells count=154 2024-12-04T19:13:58,551 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 97532d2585300ca10ffc4f6d5dda82c5 3/3 column families, dataSize=71.56 KB heapSize=195.34 KB 2024-12-04T19:13:58,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-04T19:13:58,548 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1733339638204, max timestamp=1733339638544Num uniques 1154; 2024-12-04T19:13:58,553 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=52850, new segment data size=7050, suffix heap size=147984,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1154, new segment cells count=154 2024-12-04T19:13:58,560 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97532d2585300ca10ffc4f6d5dda82c5, store=A 2024-12-04T19:13:58,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T19:13:58,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97532d2585300ca10ffc4f6d5dda82c5, store=B 2024-12-04T19:13:58,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T19:13:58,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97532d2585300ca10ffc4f6d5dda82c5, store=C 2024-12-04T19:13:58,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T19:13:58,562 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-04T19:13:58,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:13:58,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. as already flushing 2024-12-04T19:13:58,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:13:58,563 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:13:58,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:13:58,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:13:58,588 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/A/8e5c7f579ecf46d2ae3b80c37a741dbb is 50, key is test_row_0/A:col10/1733339638549/Put/seqid=0 2024-12-04T19:13:58,597 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 97532d2585300ca10ffc4f6d5dda82c5#B#compaction#11 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:13:58,598 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/B/5b523375fe0f463cbcae448e4cb1a547 is 50, key is test_row_0/B:col10/1733339637194/Put/seqid=0 2024-12-04T19:13:58,599 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 97532d2585300ca10ffc4f6d5dda82c5#A#compaction#9 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:13:58,600 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/A/03a3da469b554415b4cdcf6bfcf2fa84 is 50, key is test_row_0/A:col10/1733339637194/Put/seqid=0 2024-12-04T19:13:58,626 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733339638576, max timestamp=1733339638622Num uniques 859; 2024-12-04T19:13:58,626 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733339638576, max timestamp=1733339638622Num uniques 859; 2024-12-04T19:13:58,626 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T19:13:58,626 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T19:13:58,629 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733339638576, max timestamp=1733339638622Num uniques 859; 2024-12-04T19:13:58,629 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T19:13:58,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741847_1023 (size=12104) 2024-12-04T19:13:58,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741848_1024 (size=12104) 2024-12-04T19:13:58,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741846_1022 (size=12151) 2024-12-04T19:13:58,657 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/B/5b523375fe0f463cbcae448e4cb1a547 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/5b523375fe0f463cbcae448e4cb1a547 2024-12-04T19:13:58,684 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 97532d2585300ca10ffc4f6d5dda82c5/B of 97532d2585300ca10ffc4f6d5dda82c5 into 5b523375fe0f463cbcae448e4cb1a547(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:13:58,684 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 97532d2585300ca10ffc4f6d5dda82c5: 2024-12-04T19:13:58,687 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., storeName=97532d2585300ca10ffc4f6d5dda82c5/B, priority=13, startTime=1733339638515; duration=0sec 2024-12-04T19:13:58,688 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T19:13:58,688 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 97532d2585300ca10ffc4f6d5dda82c5:B 2024-12-04T19:13:58,688 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T19:13:58,692 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T19:13:58,692 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1541): 97532d2585300ca10ffc4f6d5dda82c5/C is initiating minor compaction (all files) 2024-12-04T19:13:58,692 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 97532d2585300ca10ffc4f6d5dda82c5/C in TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:13:58,692 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/fb6347ed02a843fabb95ecbad719fb4f, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/7c9469d1b25141b3a123563a45af1c09, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/4f39b560c9fc4f1e8f366336a570451c] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp, totalSize=35.2 K 2024-12-04T19:13:58,693 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting fb6347ed02a843fabb95ecbad719fb4f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733339636408 2024-12-04T19:13:58,694 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733339638676, max timestamp=1733339638690Num uniques 1068; 2024-12-04T19:13:58,694 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c9469d1b25141b3a123563a45af1c09, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1733339636741 2024-12-04T19:13:58,694 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-12-04T19:13:58,695 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733339638676, max timestamp=1733339638690Num uniques 1068; 2024-12-04T19:13:58,695 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-12-04T19:13:58,697 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f39b560c9fc4f1e8f366336a570451c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1733339637194 2024-12-04T19:13:58,698 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733339638676, max timestamp=1733339638690Num uniques 1068; 2024-12-04T19:13:58,698 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-12-04T19:13:58,718 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-04T19:13:58,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:13:58,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. as already flushing 2024-12-04T19:13:58,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:13:58,719 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:13:58,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:13:58,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:13:58,742 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 97532d2585300ca10ffc4f6d5dda82c5#C#compaction#12 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:13:58,743 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/C/d5790d0e114f4e3eb796bfb274c528d8 is 50, key is test_row_0/C:col10/1733339637194/Put/seqid=0 2024-12-04T19:13:58,752 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1733339638734, max timestamp=1733339638748Num uniques 1027; 2024-12-04T19:13:58,753 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=12682, suffix heap size=134776,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=277 2024-12-04T19:13:58,753 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1733339638734, max timestamp=1733339638748Num uniques 1027; 2024-12-04T19:13:58,753 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=12682, suffix heap size=134776,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=277 2024-12-04T19:13:58,753 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1733339638734, max timestamp=1733339638748Num uniques 1027; 2024-12-04T19:13:58,753 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=12682, suffix heap size=134776,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=277 2024-12-04T19:13:58,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741849_1025 (size=12104) 2024-12-04T19:13:58,806 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=386, cellSize=17676, totalHeapSize=40384, min timestamp=1733339638777, max timestamp=1733339638800Num uniques 1136; 2024-12-04T19:13:58,806 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=17676, suffix heap size=146112,new segment heap size=40384 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=386 2024-12-04T19:13:58,806 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=386, cellSize=17676, totalHeapSize=40384, min timestamp=1733339638777, max timestamp=1733339638800Num uniques 1136; 2024-12-04T19:13:58,806 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=17676, suffix heap size=146112,new segment heap size=40384 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=386 2024-12-04T19:13:58,808 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/C/d5790d0e114f4e3eb796bfb274c528d8 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/d5790d0e114f4e3eb796bfb274c528d8 2024-12-04T19:13:58,813 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=386, cellSize=17676, totalHeapSize=40384, min timestamp=1733339638777, max timestamp=1733339638800Num uniques 1136; 2024-12-04T19:13:58,813 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=17676, suffix heap size=146112,new segment heap size=40384 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=386 2024-12-04T19:13:58,831 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 97532d2585300ca10ffc4f6d5dda82c5/C of 97532d2585300ca10ffc4f6d5dda82c5 into d5790d0e114f4e3eb796bfb274c528d8(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:13:58,831 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 97532d2585300ca10ffc4f6d5dda82c5: 2024-12-04T19:13:58,831 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., storeName=97532d2585300ca10ffc4f6d5dda82c5/C, priority=13, startTime=1733339638518; duration=0sec 2024-12-04T19:13:58,832 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:13:58,832 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 97532d2585300ca10ffc4f6d5dda82c5:C 2024-12-04T19:13:58,849 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:13:58,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57600 deadline: 1733339698837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:13:58,850 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:13:58,850 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:58,850 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:58,850 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:13:58,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57662 deadline: 1733339698840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:13:58,851 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:58,851 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:58,851 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:13:58,851 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:13:58,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57588 deadline: 1733339698842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:13:58,852 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:58,852 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:58,852 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:13:58,852 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:13:58,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57646 deadline: 1733339698843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:13:58,853 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:58,854 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:58,854 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:13:58,856 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:13:58,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57632 deadline: 1733339698849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:13:58,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:58,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:58,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:13:58,881 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-04T19:13:58,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:13:58,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. as already flushing 2024-12-04T19:13:58,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:13:58,882 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:13:58,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:13:58,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:13:58,952 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:13:58,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57600 deadline: 1733339698952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:13:58,954 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:58,954 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:58,954 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:13:58,965 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:13:58,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57588 deadline: 1733339698963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:13:58,966 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:58,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:13:58,967 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:58,967 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:13:58,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57662 deadline: 1733339698963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:13:58,967 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:13:58,967 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:58,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57632 deadline: 1733339698963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:13:58,967 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:13:58,967 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:58,968 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:13:58,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57646 deadline: 1733339698963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:13:58,968 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:58,968 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:58,968 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:13:58,968 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:58,969 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:58,969 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:13:59,039 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-04T19:13:59,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:13:59,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. as already flushing 2024-12-04T19:13:59,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:13:59,040 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:13:59,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:13:59,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:13:59,046 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/A/8e5c7f579ecf46d2ae3b80c37a741dbb 2024-12-04T19:13:59,059 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/A/03a3da469b554415b4cdcf6bfcf2fa84 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/03a3da469b554415b4cdcf6bfcf2fa84 2024-12-04T19:13:59,079 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 97532d2585300ca10ffc4f6d5dda82c5/A of 97532d2585300ca10ffc4f6d5dda82c5 into 03a3da469b554415b4cdcf6bfcf2fa84(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:13:59,079 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 97532d2585300ca10ffc4f6d5dda82c5: 2024-12-04T19:13:59,079 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., storeName=97532d2585300ca10ffc4f6d5dda82c5/A, priority=13, startTime=1733339638508; duration=0sec 2024-12-04T19:13:59,080 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:13:59,080 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 97532d2585300ca10ffc4f6d5dda82c5:A 2024-12-04T19:13:59,082 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/B/b3b4003cb2f44c5cb8ff6b5cefbf088f is 50, key is test_row_0/B:col10/1733339638549/Put/seqid=0 2024-12-04T19:13:59,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741850_1026 (size=12151) 2024-12-04T19:13:59,111 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/B/b3b4003cb2f44c5cb8ff6b5cefbf088f 2024-12-04T19:13:59,122 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-04T19:13:59,142 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/C/8ebee7e51f544217a47e85bf92e8f86a is 50, key is test_row_0/C:col10/1733339638549/Put/seqid=0 2024-12-04T19:13:59,165 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:13:59,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57600 deadline: 1733339699162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:13:59,167 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:59,167 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:59,167 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:13:59,174 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:13:59,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57588 deadline: 1733339699174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:13:59,175 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:13:59,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57662 deadline: 1733339699174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:13:59,176 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:13:59,176 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:59,176 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:59,176 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:59,176 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:59,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57632 deadline: 1733339699174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:13:59,176 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:13:59,176 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:13:59,177 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:59,177 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:13:59,177 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:59,177 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:13:59,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57646 deadline: 1733339699175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:13:59,177 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:59,177 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:59,178 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:13:59,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741851_1027 (size=12151) 2024-12-04T19:13:59,195 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-04T19:13:59,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:13:59,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. as already flushing 2024-12-04T19:13:59,196 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/C/8ebee7e51f544217a47e85bf92e8f86a 2024-12-04T19:13:59,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:13:59,196 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:13:59,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:13:59,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:13:59,209 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/A/8e5c7f579ecf46d2ae3b80c37a741dbb as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/8e5c7f579ecf46d2ae3b80c37a741dbb 2024-12-04T19:13:59,221 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/8e5c7f579ecf46d2ae3b80c37a741dbb, entries=150, sequenceid=166, filesize=11.9 K 2024-12-04T19:13:59,224 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/B/b3b4003cb2f44c5cb8ff6b5cefbf088f as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/b3b4003cb2f44c5cb8ff6b5cefbf088f 2024-12-04T19:13:59,238 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/b3b4003cb2f44c5cb8ff6b5cefbf088f, entries=150, sequenceid=166, filesize=11.9 K 2024-12-04T19:13:59,240 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/C/8ebee7e51f544217a47e85bf92e8f86a as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/8ebee7e51f544217a47e85bf92e8f86a 2024-12-04T19:13:59,252 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/8ebee7e51f544217a47e85bf92e8f86a, entries=150, sequenceid=166, filesize=11.9 K 2024-12-04T19:13:59,254 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~86.58 KB/88656, currentSize=161.02 KB/164880 for 97532d2585300ca10ffc4f6d5dda82c5 in 702ms, sequenceid=166, compaction requested=false 2024-12-04T19:13:59,254 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 97532d2585300ca10ffc4f6d5dda82c5: 2024-12-04T19:13:59,255 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-04T19:13:59,256 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-04T19:13:59,258 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-04T19:13:59,258 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-04T19:13:59,259 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-04T19:13:59,259 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-04T19:13:59,261 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-04T19:13:59,261 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-12-04T19:13:59,351 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-04T19:13:59,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:13:59,352 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 97532d2585300ca10ffc4f6d5dda82c5 3/3 column families, dataSize=161.02 KB heapSize=443.39 KB 2024-12-04T19:13:59,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97532d2585300ca10ffc4f6d5dda82c5, store=A 2024-12-04T19:13:59,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T19:13:59,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97532d2585300ca10ffc4f6d5dda82c5, store=B 2024-12-04T19:13:59,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T19:13:59,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97532d2585300ca10ffc4f6d5dda82c5, store=C 2024-12-04T19:13:59,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T19:13:59,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/A/4eec874e412f4284be5c88e2b57498f9 is 50, key is test_row_0/A:col10/1733339638840/Put/seqid=0 2024-12-04T19:13:59,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741852_1028 (size=12151) 2024-12-04T19:13:59,401 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/A/4eec874e412f4284be5c88e2b57498f9 2024-12-04T19:13:59,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/B/368192e7f4d24b35ba5878a7d6687385 is 50, key is test_row_0/B:col10/1733339638840/Put/seqid=0 2024-12-04T19:13:59,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741853_1029 (size=12151) 2024-12-04T19:13:59,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(8855): Flush requested on 97532d2585300ca10ffc4f6d5dda82c5 2024-12-04T19:13:59,485 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. as already flushing 2024-12-04T19:13:59,495 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:13:59,495 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:13:59,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57588 deadline: 1733339699489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:13:59,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57600 deadline: 1733339699486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:13:59,496 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:59,496 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:59,496 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:13:59,496 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:59,496 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:59,496 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:13:59,496 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:13:59,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57646 deadline: 1733339699491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:13:59,497 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:59,497 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:59,497 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:13:59,497 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:13:59,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57662 deadline: 1733339699494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:13:59,497 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:59,498 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:59,498 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:13:59,501 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:13:59,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57632 deadline: 1733339699496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:13:59,503 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:59,503 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:59,503 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:13:59,602 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:13:59,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57588 deadline: 1733339699602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:13:59,603 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:13:59,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57600 deadline: 1733339699602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:13:59,604 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:59,604 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:59,604 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:13:59,604 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:13:59,604 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:13:59,604 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:59,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57662 deadline: 1733339699602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:13:59,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57646 deadline: 1733339699602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:13:59,604 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:59,604 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:13:59,605 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:59,605 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:59,605 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:59,605 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:59,605 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:13:59,605 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:13:59,619 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:13:59,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57632 deadline: 1733339699616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:13:59,620 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:59,621 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:59,621 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:13:59,814 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:13:59,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57588 deadline: 1733339699811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:13:59,814 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:13:59,814 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:59,815 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:59,815 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:13:59,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57600 deadline: 1733339699811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:13:59,815 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:59,815 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:59,815 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:13:59,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:13:59,815 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:13:59,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57646 deadline: 1733339699812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:13:59,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57662 deadline: 1733339699812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:13:59,816 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:59,816 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:59,816 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:13:59,816 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:59,816 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:59,816 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:13:59,832 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:13:59,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57632 deadline: 1733339699832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:13:59,833 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:59,833 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:13:59,833 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:13:59,856 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/B/368192e7f4d24b35ba5878a7d6687385 2024-12-04T19:13:59,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/C/c7ebb8f4c6c24013908f3453a997bb7a is 50, key is test_row_0/C:col10/1733339638840/Put/seqid=0 2024-12-04T19:13:59,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741854_1030 (size=12151) 2024-12-04T19:14:00,125 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:00,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57588 deadline: 1733339700122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:00,126 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:00,126 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:00,126 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:00,126 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:00,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57662 deadline: 1733339700122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:00,127 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:00,127 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:00,127 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:00,127 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:00,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57600 deadline: 1733339700123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:00,127 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:00,128 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:00,128 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:00,128 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:00,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57646 deadline: 1733339700123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:00,128 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:00,128 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:00,128 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:00,148 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:00,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57632 deadline: 1733339700142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:00,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:00,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:00,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:00,304 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/C/c7ebb8f4c6c24013908f3453a997bb7a 2024-12-04T19:14:00,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/A/4eec874e412f4284be5c88e2b57498f9 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/4eec874e412f4284be5c88e2b57498f9 2024-12-04T19:14:00,328 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/4eec874e412f4284be5c88e2b57498f9, entries=150, sequenceid=257, filesize=11.9 K 2024-12-04T19:14:00,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/B/368192e7f4d24b35ba5878a7d6687385 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/368192e7f4d24b35ba5878a7d6687385 2024-12-04T19:14:00,343 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/368192e7f4d24b35ba5878a7d6687385, entries=150, sequenceid=257, filesize=11.9 K 2024-12-04T19:14:00,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/C/c7ebb8f4c6c24013908f3453a997bb7a as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/c7ebb8f4c6c24013908f3453a997bb7a 2024-12-04T19:14:00,362 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/c7ebb8f4c6c24013908f3453a997bb7a, entries=150, sequenceid=257, filesize=11.9 K 2024-12-04T19:14:00,363 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~443.34 KB/453984, currentSize=33.54 KB/34350 for 97532d2585300ca10ffc4f6d5dda82c5 in 1011ms, sequenceid=257, compaction requested=true 2024-12-04T19:14:00,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 97532d2585300ca10ffc4f6d5dda82c5: 2024-12-04T19:14:00,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:00,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-12-04T19:14:00,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-12-04T19:14:00,370 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-04T19:14:00,370 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.9650 sec 2024-12-04T19:14:00,375 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 3.9920 sec 2024-12-04T19:14:00,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-04T19:14:00,582 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T19:14:00,583 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T19:14:00,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-12-04T19:14:00,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-04T19:14:00,588 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T19:14:00,591 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T19:14:00,591 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T19:14:00,636 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 97532d2585300ca10ffc4f6d5dda82c5 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-04T19:14:00,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(8855): Flush requested on 97532d2585300ca10ffc4f6d5dda82c5 2024-12-04T19:14:00,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97532d2585300ca10ffc4f6d5dda82c5, store=A 2024-12-04T19:14:00,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:14:00,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97532d2585300ca10ffc4f6d5dda82c5, store=B 2024-12-04T19:14:00,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:14:00,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97532d2585300ca10ffc4f6d5dda82c5, store=C 2024-12-04T19:14:00,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:14:00,647 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/A/574fea5039fc4ce88a7a56c9c5d34f8c is 50, key is test_row_0/A:col10/1733339640634/Put/seqid=0 2024-12-04T19:14:00,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741855_1031 (size=17181) 2024-12-04T19:14:00,688 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733339640667, max timestamp=1733339640685Num uniques 859; 2024-12-04T19:14:00,688 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733339640667, max timestamp=1733339640685Num uniques 859; 2024-12-04T19:14:00,688 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733339640667, max timestamp=1733339640685Num uniques 859; 2024-12-04T19:14:00,688 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-04T19:14:00,688 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-04T19:14:00,688 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-04T19:14:00,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-04T19:14:00,720 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:00,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57600 deadline: 1733339700711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:00,722 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:00,722 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:00,722 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:00,722 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:00,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57632 deadline: 1733339700713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:00,723 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:00,723 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:00,723 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:00,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:00,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57662 deadline: 1733339700714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:00,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:00,726 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:00,726 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:00,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57646 deadline: 1733339700716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:00,727 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:00,727 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:00,727 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:00,727 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:00,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:00,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57588 deadline: 1733339700721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:00,729 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:00,729 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:00,729 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:00,746 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-04T19:14:00,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:00,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. as already flushing 2024-12-04T19:14:00,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:00,747 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:00,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:00,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:00,833 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:00,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57600 deadline: 1733339700832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:00,834 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:00,834 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:00,834 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:00,834 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:00,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57632 deadline: 1733339700832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:00,834 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:00,835 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:00,835 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:00,834 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:00,835 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:00,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57662 deadline: 1733339700833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:00,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57588 deadline: 1733339700833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:00,835 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:00,835 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:00,835 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:00,835 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:00,835 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:00,835 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:00,835 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:00,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57646 deadline: 1733339700833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:00,836 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:00,836 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:00,836 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:00,901 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-04T19:14:00,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:00,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. as already flushing 2024-12-04T19:14:00,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:00,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-04T19:14:00,902 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:00,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:00,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:01,043 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:01,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57600 deadline: 1733339701042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:01,044 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:01,044 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:01,044 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:01,044 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:01,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57632 deadline: 1733339701042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:01,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:01,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:01,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:01,045 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:01,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57662 deadline: 1733339701043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:01,045 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:01,045 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:01,046 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:01,045 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:01,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57588 deadline: 1733339701043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:01,046 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:01,046 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:01,046 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:01,046 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:01,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57646 deadline: 1733339701043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:01,047 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:01,047 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:01,047 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:01,056 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-04T19:14:01,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:01,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. as already flushing 2024-12-04T19:14:01,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:01,056 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:01,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:01,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:01,077 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=268 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/A/574fea5039fc4ce88a7a56c9c5d34f8c 2024-12-04T19:14:01,090 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/B/cdf6d60e739f4885bb1ae73c49590444 is 50, key is test_row_0/B:col10/1733339640634/Put/seqid=0 2024-12-04T19:14:01,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741856_1032 (size=12301) 2024-12-04T19:14:01,210 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-04T19:14:01,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:01,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. as already flushing 2024-12-04T19:14:01,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:01,211 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:01,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:01,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-04T19:14:01,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:01,354 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:01,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57600 deadline: 1733339701352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:01,355 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:01,355 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:01,355 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:01,355 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:01,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57632 deadline: 1733339701352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:01,356 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:01,356 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:01,356 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:01,356 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:01,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57662 deadline: 1733339701352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:01,357 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:01,357 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:01,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57588 deadline: 1733339701352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:01,357 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:01,357 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:01,357 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:01,357 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:01,357 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:01,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57646 deadline: 1733339701352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:01,357 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:01,358 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:01,358 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:01,359 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:01,364 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-04T19:14:01,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:01,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. as already flushing 2024-12-04T19:14:01,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:01,365 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:01,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:01,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:01,497 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=268 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/B/cdf6d60e739f4885bb1ae73c49590444 2024-12-04T19:14:01,512 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/C/cdaa2070afb6475b8e0819d95028b3d8 is 50, key is test_row_0/C:col10/1733339640634/Put/seqid=0 2024-12-04T19:14:01,519 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-04T19:14:01,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:01,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. as already flushing 2024-12-04T19:14:01,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:01,519 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:01,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:01,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:01,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741857_1033 (size=12301) 2024-12-04T19:14:01,674 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-04T19:14:01,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:01,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. as already flushing 2024-12-04T19:14:01,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:01,674 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:01,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:01,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:01,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-04T19:14:01,828 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-04T19:14:01,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:01,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. as already flushing 2024-12-04T19:14:01,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:01,828 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:01,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:01,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:01,865 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:01,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57600 deadline: 1733339701861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:01,866 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:01,866 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:01,866 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:01,867 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:01,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57632 deadline: 1733339701862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:01,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:01,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:01,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:01,867 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:01,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57646 deadline: 1733339701862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:01,868 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:01,868 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:01,868 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:01,874 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:01,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57662 deadline: 1733339701872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:01,874 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:01,874 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:01,875 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:01,875 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:01,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57588 deadline: 1733339701872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:01,875 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:01,875 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:01,875 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:01,923 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=268 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/C/cdaa2070afb6475b8e0819d95028b3d8 2024-12-04T19:14:01,933 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/A/574fea5039fc4ce88a7a56c9c5d34f8c as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/574fea5039fc4ce88a7a56c9c5d34f8c 2024-12-04T19:14:01,942 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/574fea5039fc4ce88a7a56c9c5d34f8c, entries=250, sequenceid=268, filesize=16.8 K 2024-12-04T19:14:01,944 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/B/cdf6d60e739f4885bb1ae73c49590444 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/cdf6d60e739f4885bb1ae73c49590444 2024-12-04T19:14:01,953 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/cdf6d60e739f4885bb1ae73c49590444, entries=150, sequenceid=268, filesize=12.0 K 2024-12-04T19:14:01,955 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/C/cdaa2070afb6475b8e0819d95028b3d8 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/cdaa2070afb6475b8e0819d95028b3d8 2024-12-04T19:14:01,968 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/cdaa2070afb6475b8e0819d95028b3d8, entries=150, sequenceid=268, filesize=12.0 K 2024-12-04T19:14:01,969 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 97532d2585300ca10ffc4f6d5dda82c5 in 1333ms, sequenceid=268, compaction requested=true 2024-12-04T19:14:01,970 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 97532d2585300ca10ffc4f6d5dda82c5: 2024-12-04T19:14:01,970 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T19:14:01,970 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 97532d2585300ca10ffc4f6d5dda82c5:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T19:14:01,970 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:14:01,971 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T19:14:01,971 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 97532d2585300ca10ffc4f6d5dda82c5:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T19:14:01,971 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:14:01,971 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 97532d2585300ca10ffc4f6d5dda82c5:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T19:14:01,971 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T19:14:01,973 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53587 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T19:14:01,973 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1541): 97532d2585300ca10ffc4f6d5dda82c5/A is initiating minor compaction (all files) 2024-12-04T19:14:01,973 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 97532d2585300ca10ffc4f6d5dda82c5/A in TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:01,973 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/03a3da469b554415b4cdcf6bfcf2fa84, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/8e5c7f579ecf46d2ae3b80c37a741dbb, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/4eec874e412f4284be5c88e2b57498f9, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/574fea5039fc4ce88a7a56c9c5d34f8c] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp, totalSize=52.3 K 2024-12-04T19:14:01,974 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48707 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T19:14:01,974 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1541): 97532d2585300ca10ffc4f6d5dda82c5/B is initiating minor compaction (all files) 2024-12-04T19:14:01,974 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 97532d2585300ca10ffc4f6d5dda82c5/B in TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:01,974 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/5b523375fe0f463cbcae448e4cb1a547, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/b3b4003cb2f44c5cb8ff6b5cefbf088f, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/368192e7f4d24b35ba5878a7d6687385, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/cdf6d60e739f4885bb1ae73c49590444] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp, totalSize=47.6 K 2024-12-04T19:14:01,975 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 03a3da469b554415b4cdcf6bfcf2fa84, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1733339637194 2024-12-04T19:14:01,975 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b523375fe0f463cbcae448e4cb1a547, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1733339637194 2024-12-04T19:14:01,976 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8e5c7f579ecf46d2ae3b80c37a741dbb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1733339638212 2024-12-04T19:14:01,979 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4eec874e412f4284be5c88e2b57498f9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1733339638839 2024-12-04T19:14:01,980 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting b3b4003cb2f44c5cb8ff6b5cefbf088f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1733339638212 2024-12-04T19:14:01,981 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 368192e7f4d24b35ba5878a7d6687385, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1733339638839 2024-12-04T19:14:01,981 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 574fea5039fc4ce88a7a56c9c5d34f8c, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1733339639489 2024-12-04T19:14:01,982 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting cdf6d60e739f4885bb1ae73c49590444, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1733339640633 2024-12-04T19:14:01,985 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-04T19:14:01,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:01,985 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 97532d2585300ca10ffc4f6d5dda82c5 3/3 column families, dataSize=134.18 KB heapSize=370.92 KB 2024-12-04T19:14:01,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97532d2585300ca10ffc4f6d5dda82c5, store=A 2024-12-04T19:14:01,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T19:14:01,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97532d2585300ca10ffc4f6d5dda82c5, store=B 2024-12-04T19:14:01,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T19:14:01,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97532d2585300ca10ffc4f6d5dda82c5, store=C 2024-12-04T19:14:01,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T19:14:01,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/A/d2f5d28c3461429ea2b98166e4dd052a is 50, key is test_row_0/A:col10/1733339640712/Put/seqid=0 2024-12-04T19:14:02,017 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 97532d2585300ca10ffc4f6d5dda82c5#A#compaction#22 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:14:02,018 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/A/8c901c31e3ca40fc811b6c14f3bfe19e is 50, key is test_row_0/A:col10/1733339640634/Put/seqid=0 2024-12-04T19:14:02,050 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 97532d2585300ca10ffc4f6d5dda82c5#B#compaction#23 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:14:02,050 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/B/5e9d422ea79b4da2b74148d91f955530 is 50, key is test_row_0/B:col10/1733339640634/Put/seqid=0 2024-12-04T19:14:02,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741858_1034 (size=12301) 2024-12-04T19:14:02,068 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=302 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/A/d2f5d28c3461429ea2b98166e4dd052a 2024-12-04T19:14:02,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741859_1035 (size=12541) 2024-12-04T19:14:02,090 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/A/8c901c31e3ca40fc811b6c14f3bfe19e as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/8c901c31e3ca40fc811b6c14f3bfe19e 2024-12-04T19:14:02,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741860_1036 (size=12541) 2024-12-04T19:14:02,110 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 97532d2585300ca10ffc4f6d5dda82c5/A of 97532d2585300ca10ffc4f6d5dda82c5 into 8c901c31e3ca40fc811b6c14f3bfe19e(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:14:02,110 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 97532d2585300ca10ffc4f6d5dda82c5: 2024-12-04T19:14:02,110 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., storeName=97532d2585300ca10ffc4f6d5dda82c5/A, priority=12, startTime=1733339641970; duration=0sec 2024-12-04T19:14:02,110 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T19:14:02,114 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 97532d2585300ca10ffc4f6d5dda82c5:A 2024-12-04T19:14:02,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/B/026a0339a5094f14a3405e8577132fcb is 50, key is test_row_0/B:col10/1733339640712/Put/seqid=0 2024-12-04T19:14:02,114 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T19:14:02,130 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/B/5e9d422ea79b4da2b74148d91f955530 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/5e9d422ea79b4da2b74148d91f955530 2024-12-04T19:14:02,131 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48707 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T19:14:02,132 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1541): 97532d2585300ca10ffc4f6d5dda82c5/C is initiating minor compaction (all files) 2024-12-04T19:14:02,132 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 97532d2585300ca10ffc4f6d5dda82c5/C in TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:02,132 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/d5790d0e114f4e3eb796bfb274c528d8, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/8ebee7e51f544217a47e85bf92e8f86a, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/c7ebb8f4c6c24013908f3453a997bb7a, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/cdaa2070afb6475b8e0819d95028b3d8] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp, totalSize=47.6 K 2024-12-04T19:14:02,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741861_1037 (size=12301) 2024-12-04T19:14:02,133 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting d5790d0e114f4e3eb796bfb274c528d8, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1733339637194 2024-12-04T19:14:02,134 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8ebee7e51f544217a47e85bf92e8f86a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1733339638212 2024-12-04T19:14:02,134 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting c7ebb8f4c6c24013908f3453a997bb7a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1733339638839 2024-12-04T19:14:02,135 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting cdaa2070afb6475b8e0819d95028b3d8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1733339640633 2024-12-04T19:14:02,137 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=302 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/B/026a0339a5094f14a3405e8577132fcb 2024-12-04T19:14:02,151 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 97532d2585300ca10ffc4f6d5dda82c5/B of 97532d2585300ca10ffc4f6d5dda82c5 into 5e9d422ea79b4da2b74148d91f955530(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:14:02,151 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 97532d2585300ca10ffc4f6d5dda82c5: 2024-12-04T19:14:02,151 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., storeName=97532d2585300ca10ffc4f6d5dda82c5/B, priority=12, startTime=1733339641970; duration=0sec 2024-12-04T19:14:02,152 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:14:02,152 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 97532d2585300ca10ffc4f6d5dda82c5:B 2024-12-04T19:14:02,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/C/a5e5b1605f684d3d92dca4c244c9e255 is 50, key is test_row_0/C:col10/1733339640712/Put/seqid=0 2024-12-04T19:14:02,167 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 97532d2585300ca10ffc4f6d5dda82c5#C#compaction#26 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:14:02,168 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/C/522cf64f0cbb4b04897c7560cf637917 is 50, key is test_row_0/C:col10/1733339640634/Put/seqid=0 2024-12-04T19:14:02,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741862_1038 (size=12301) 2024-12-04T19:14:02,172 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=302 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/C/a5e5b1605f684d3d92dca4c244c9e255 2024-12-04T19:14:02,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/A/d2f5d28c3461429ea2b98166e4dd052a as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/d2f5d28c3461429ea2b98166e4dd052a 2024-12-04T19:14:02,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741863_1039 (size=12541) 2024-12-04T19:14:02,190 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/d2f5d28c3461429ea2b98166e4dd052a, entries=150, sequenceid=302, filesize=12.0 K 2024-12-04T19:14:02,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/B/026a0339a5094f14a3405e8577132fcb as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/026a0339a5094f14a3405e8577132fcb 2024-12-04T19:14:02,205 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/026a0339a5094f14a3405e8577132fcb, entries=150, sequenceid=302, filesize=12.0 K 2024-12-04T19:14:02,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/C/a5e5b1605f684d3d92dca4c244c9e255 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/a5e5b1605f684d3d92dca4c244c9e255 2024-12-04T19:14:02,216 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/a5e5b1605f684d3d92dca4c244c9e255, entries=150, sequenceid=302, filesize=12.0 K 2024-12-04T19:14:02,217 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~370.88 KB/379776, currentSize=0 B/0 for 97532d2585300ca10ffc4f6d5dda82c5 in 232ms, sequenceid=302, compaction requested=false 2024-12-04T19:14:02,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 97532d2585300ca10ffc4f6d5dda82c5: 2024-12-04T19:14:02,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:02,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-12-04T19:14:02,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-12-04T19:14:02,224 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-12-04T19:14:02,224 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6280 sec 2024-12-04T19:14:02,227 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 1.6420 sec 2024-12-04T19:14:02,592 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/C/522cf64f0cbb4b04897c7560cf637917 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/522cf64f0cbb4b04897c7560cf637917 2024-12-04T19:14:02,622 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 97532d2585300ca10ffc4f6d5dda82c5/C of 97532d2585300ca10ffc4f6d5dda82c5 into 522cf64f0cbb4b04897c7560cf637917(size=12.2 K), total size for store is 24.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:14:02,622 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 97532d2585300ca10ffc4f6d5dda82c5: 2024-12-04T19:14:02,622 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., storeName=97532d2585300ca10ffc4f6d5dda82c5/C, priority=12, startTime=1733339641971; duration=0sec 2024-12-04T19:14:02,622 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:14:02,622 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 97532d2585300ca10ffc4f6d5dda82c5:C 2024-12-04T19:14:02,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-04T19:14:02,732 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T19:14:02,733 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T19:14:02,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-12-04T19:14:02,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-04T19:14:02,737 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T19:14:02,738 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T19:14:02,739 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T19:14:02,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-04T19:14:02,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(8855): Flush requested on 97532d2585300ca10ffc4f6d5dda82c5 2024-12-04T19:14:02,889 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 97532d2585300ca10ffc4f6d5dda82c5 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-04T19:14:02,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97532d2585300ca10ffc4f6d5dda82c5, store=A 2024-12-04T19:14:02,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:14:02,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97532d2585300ca10ffc4f6d5dda82c5, store=B 2024-12-04T19:14:02,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:14:02,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97532d2585300ca10ffc4f6d5dda82c5, store=C 2024-12-04T19:14:02,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:14:02,893 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-04T19:14:02,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:02,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. as already flushing 2024-12-04T19:14:02,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:02,893 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:02,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:02,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:02,897 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/A/e43a20dbb858412781826acb0d484fa8 is 50, key is test_row_0/A:col10/1733339642887/Put/seqid=0 2024-12-04T19:14:02,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741864_1040 (size=19621) 2024-12-04T19:14:02,925 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733339642893, max timestamp=1733339642921Num uniques 859; 2024-12-04T19:14:02,925 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733339642893, max timestamp=1733339642921Num uniques 859; 2024-12-04T19:14:02,925 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-04T19:14:02,925 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-04T19:14:02,925 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733339642893, max timestamp=1733339642921Num uniques 859; 2024-12-04T19:14:02,926 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-04T19:14:02,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:02,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57646 deadline: 1733339702943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:02,950 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:02,950 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:02,951 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:02,953 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:02,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57662 deadline: 1733339702946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:02,955 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:02,955 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:02,955 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:02,955 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:02,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57588 deadline: 1733339702950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:02,955 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:02,955 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:02,956 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:02,955 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:02,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57600 deadline: 1733339702954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:02,956 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:02,956 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:02,956 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:02,956 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:02,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57632 deadline: 1733339702954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:02,957 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:02,957 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:02,957 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:03,047 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-04T19:14:03,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:03,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. as already flushing 2024-12-04T19:14:03,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:03,047 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:03,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:03,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:03,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-04T19:14:03,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:03,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57646 deadline: 1733339703061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:03,064 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:03,064 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:03,064 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:03,064 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:03,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57662 deadline: 1733339703062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:03,064 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:03,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57588 deadline: 1733339703062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:03,065 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:03,065 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:03,065 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:03,065 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:03,065 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:03,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57600 deadline: 1733339703062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:03,065 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:03,065 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:03,065 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:03,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57632 deadline: 1733339703062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:03,065 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:03,065 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:03,065 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:03,065 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:03,066 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:03,066 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:03,200 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-04T19:14:03,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:03,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. as already flushing 2024-12-04T19:14:03,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:03,201 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:03,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:03,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:03,273 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:03,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57646 deadline: 1733339703272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:03,273 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:03,274 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:03,273 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:03,274 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:03,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57662 deadline: 1733339703272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:03,274 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:03,274 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:03,274 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:03,275 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:03,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57588 deadline: 1733339703272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:03,276 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:03,276 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:03,276 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:03,276 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:03,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57632 deadline: 1733339703273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:03,276 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:03,276 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:03,276 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:03,276 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:03,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57600 deadline: 1733339703273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:03,277 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:03,277 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:03,277 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:03,318 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/A/e43a20dbb858412781826acb0d484fa8 2024-12-04T19:14:03,331 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/B/4540577f834e43b88493db7c299d8aa9 is 50, key is test_row_0/B:col10/1733339642887/Put/seqid=0 2024-12-04T19:14:03,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741865_1041 (size=12301) 2024-12-04T19:14:03,354 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-04T19:14:03,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:03,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. as already flushing 2024-12-04T19:14:03,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:03,354 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:03,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:03,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:03,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-04T19:14:03,508 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-04T19:14:03,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:03,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. as already flushing 2024-12-04T19:14:03,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:03,509 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:03,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:03,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:03,582 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:03,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57646 deadline: 1733339703581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:03,583 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:03,583 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:03,583 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:03,583 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:03,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57662 deadline: 1733339703582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:03,584 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:03,584 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:03,584 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:03,584 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:03,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57632 deadline: 1733339703582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:03,584 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:03,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57600 deadline: 1733339703582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:03,585 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:03,585 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:03,585 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:03,585 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:03,585 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:03,585 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:03,585 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:03,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57588 deadline: 1733339703582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:03,586 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:03,586 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:03,586 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:03,665 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-04T19:14:03,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:03,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. as already flushing 2024-12-04T19:14:03,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:03,666 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:03,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:03,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:03,738 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/B/4540577f834e43b88493db7c299d8aa9 2024-12-04T19:14:03,790 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/C/34ba4ba4bd474ee4aa115d1dfbf9ed7e is 50, key is test_row_0/C:col10/1733339642887/Put/seqid=0 2024-12-04T19:14:03,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741866_1042 (size=12301) 2024-12-04T19:14:03,797 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/C/34ba4ba4bd474ee4aa115d1dfbf9ed7e 2024-12-04T19:14:03,808 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/A/e43a20dbb858412781826acb0d484fa8 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/e43a20dbb858412781826acb0d484fa8 2024-12-04T19:14:03,815 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/e43a20dbb858412781826acb0d484fa8, entries=300, sequenceid=315, filesize=19.2 K 2024-12-04T19:14:03,817 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/B/4540577f834e43b88493db7c299d8aa9 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/4540577f834e43b88493db7c299d8aa9 2024-12-04T19:14:03,821 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-04T19:14:03,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:03,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. as already flushing 2024-12-04T19:14:03,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:03,821 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:03,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:03,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:03,826 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/4540577f834e43b88493db7c299d8aa9, entries=150, sequenceid=315, filesize=12.0 K 2024-12-04T19:14:03,828 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/C/34ba4ba4bd474ee4aa115d1dfbf9ed7e as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/34ba4ba4bd474ee4aa115d1dfbf9ed7e 2024-12-04T19:14:03,839 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/34ba4ba4bd474ee4aa115d1dfbf9ed7e, entries=150, sequenceid=315, filesize=12.0 K 2024-12-04T19:14:03,840 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 97532d2585300ca10ffc4f6d5dda82c5 in 951ms, sequenceid=315, compaction requested=true 2024-12-04T19:14:03,840 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 97532d2585300ca10ffc4f6d5dda82c5: 2024-12-04T19:14:03,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 97532d2585300ca10ffc4f6d5dda82c5:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T19:14:03,840 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T19:14:03,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:14:03,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 97532d2585300ca10ffc4f6d5dda82c5:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T19:14:03,841 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T19:14:03,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:14:03,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 97532d2585300ca10ffc4f6d5dda82c5:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T19:14:03,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T19:14:03,843 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 44463 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T19:14:03,843 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37143 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T19:14:03,844 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1541): 97532d2585300ca10ffc4f6d5dda82c5/A is initiating minor compaction (all files) 2024-12-04T19:14:03,844 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1541): 97532d2585300ca10ffc4f6d5dda82c5/B is initiating minor compaction (all files) 2024-12-04T19:14:03,844 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 97532d2585300ca10ffc4f6d5dda82c5/A in TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:03,844 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 97532d2585300ca10ffc4f6d5dda82c5/B in TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:03,844 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/5e9d422ea79b4da2b74148d91f955530, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/026a0339a5094f14a3405e8577132fcb, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/4540577f834e43b88493db7c299d8aa9] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp, totalSize=36.3 K 2024-12-04T19:14:03,844 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/8c901c31e3ca40fc811b6c14f3bfe19e, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/d2f5d28c3461429ea2b98166e4dd052a, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/e43a20dbb858412781826acb0d484fa8] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp, totalSize=43.4 K 2024-12-04T19:14:03,845 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e9d422ea79b4da2b74148d91f955530, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1733339640633 2024-12-04T19:14:03,845 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8c901c31e3ca40fc811b6c14f3bfe19e, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1733339640633 2024-12-04T19:14:03,846 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting d2f5d28c3461429ea2b98166e4dd052a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1733339640706 2024-12-04T19:14:03,846 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 026a0339a5094f14a3405e8577132fcb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1733339640706 2024-12-04T19:14:03,846 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting e43a20dbb858412781826acb0d484fa8, keycount=300, bloomtype=ROW, size=19.2 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1733339642875 2024-12-04T19:14:03,847 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 4540577f834e43b88493db7c299d8aa9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1733339642878 2024-12-04T19:14:03,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-04T19:14:03,878 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 97532d2585300ca10ffc4f6d5dda82c5#B#compaction#30 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:14:03,879 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/B/9cf5c91478ed4473aa486d01eec612e3 is 50, key is test_row_0/B:col10/1733339642887/Put/seqid=0 2024-12-04T19:14:03,879 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 97532d2585300ca10ffc4f6d5dda82c5#A#compaction#31 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:14:03,882 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/A/172a40f262ed4b00a129eb0eccdb9270 is 50, key is test_row_0/A:col10/1733339642887/Put/seqid=0 2024-12-04T19:14:03,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741867_1043 (size=12643) 2024-12-04T19:14:03,898 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/B/9cf5c91478ed4473aa486d01eec612e3 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/9cf5c91478ed4473aa486d01eec612e3 2024-12-04T19:14:03,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741868_1044 (size=12643) 2024-12-04T19:14:03,917 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 97532d2585300ca10ffc4f6d5dda82c5/B of 97532d2585300ca10ffc4f6d5dda82c5 into 9cf5c91478ed4473aa486d01eec612e3(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:14:03,917 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 97532d2585300ca10ffc4f6d5dda82c5: 2024-12-04T19:14:03,917 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., storeName=97532d2585300ca10ffc4f6d5dda82c5/B, priority=13, startTime=1733339643840; duration=0sec 2024-12-04T19:14:03,918 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T19:14:03,918 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 97532d2585300ca10ffc4f6d5dda82c5:B 2024-12-04T19:14:03,920 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T19:14:03,922 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37143 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T19:14:03,922 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1541): 97532d2585300ca10ffc4f6d5dda82c5/C is initiating minor compaction (all files) 2024-12-04T19:14:03,922 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 97532d2585300ca10ffc4f6d5dda82c5/C in TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:03,922 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/522cf64f0cbb4b04897c7560cf637917, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/a5e5b1605f684d3d92dca4c244c9e255, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/34ba4ba4bd474ee4aa115d1dfbf9ed7e] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp, totalSize=36.3 K 2024-12-04T19:14:03,923 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 522cf64f0cbb4b04897c7560cf637917, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1733339640633 2024-12-04T19:14:03,923 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting a5e5b1605f684d3d92dca4c244c9e255, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1733339640706 2024-12-04T19:14:03,924 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 34ba4ba4bd474ee4aa115d1dfbf9ed7e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1733339642878 2024-12-04T19:14:03,941 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 97532d2585300ca10ffc4f6d5dda82c5#C#compaction#32 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:14:03,941 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/C/262f7d96d0db4f5fb6160dcc5fd57892 is 50, key is test_row_0/C:col10/1733339642887/Put/seqid=0 2024-12-04T19:14:03,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741869_1045 (size=12643) 2024-12-04T19:14:03,976 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-04T19:14:03,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:03,977 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 97532d2585300ca10ffc4f6d5dda82c5 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-12-04T19:14:03,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97532d2585300ca10ffc4f6d5dda82c5, store=A 2024-12-04T19:14:03,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T19:14:03,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97532d2585300ca10ffc4f6d5dda82c5, store=B 2024-12-04T19:14:03,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T19:14:03,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97532d2585300ca10ffc4f6d5dda82c5, store=C 2024-12-04T19:14:03,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T19:14:03,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/A/d4de660048744e9ba0352a0b0efdce22 is 50, key is test_row_0/A:col10/1733339642947/Put/seqid=0 2024-12-04T19:14:04,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741870_1046 (size=12301) 2024-12-04T19:14:04,015 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/A/d4de660048744e9ba0352a0b0efdce22 2024-12-04T19:14:04,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/B/ac6fe4b8706746e1974dfba27f3a69f6 is 50, key is test_row_0/B:col10/1733339642947/Put/seqid=0 2024-12-04T19:14:04,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741871_1047 (size=12301) 2024-12-04T19:14:04,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(8855): Flush requested on 97532d2585300ca10ffc4f6d5dda82c5 2024-12-04T19:14:04,096 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. as already flushing 2024-12-04T19:14:04,105 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:04,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57632 deadline: 1733339704098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:04,106 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:04,106 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:04,106 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:04,110 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:04,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57600 deadline: 1733339704103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:04,111 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:04,111 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:04,111 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:04,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57588 deadline: 1733339704103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:04,111 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:04,111 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:04,111 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:04,111 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:04,111 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:04,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57662 deadline: 1733339704106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:04,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:04,112 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:04,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57646 deadline: 1733339704106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:04,112 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:04,112 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:04,113 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:04,113 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:04,113 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:04,212 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:04,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57632 deadline: 1733339704212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:04,213 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:04,213 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:04,213 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:04,222 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:04,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57600 deadline: 1733339704222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:04,223 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:04,223 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:04,223 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:04,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57588 deadline: 1733339704222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:04,223 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:04,223 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:04,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57662 deadline: 1733339704222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:04,223 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:04,223 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:04,223 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:04,224 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:04,224 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:04,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57646 deadline: 1733339704222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:04,224 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:04,224 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:04,225 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:04,225 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:04,225 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:04,316 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/A/172a40f262ed4b00a129eb0eccdb9270 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/172a40f262ed4b00a129eb0eccdb9270 2024-12-04T19:14:04,325 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 97532d2585300ca10ffc4f6d5dda82c5/A of 97532d2585300ca10ffc4f6d5dda82c5 into 172a40f262ed4b00a129eb0eccdb9270(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:14:04,325 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 97532d2585300ca10ffc4f6d5dda82c5: 2024-12-04T19:14:04,325 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., storeName=97532d2585300ca10ffc4f6d5dda82c5/A, priority=13, startTime=1733339643840; duration=0sec 2024-12-04T19:14:04,325 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:14:04,325 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 97532d2585300ca10ffc4f6d5dda82c5:A 2024-12-04T19:14:04,370 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/C/262f7d96d0db4f5fb6160dcc5fd57892 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/262f7d96d0db4f5fb6160dcc5fd57892 2024-12-04T19:14:04,379 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 97532d2585300ca10ffc4f6d5dda82c5/C of 97532d2585300ca10ffc4f6d5dda82c5 into 262f7d96d0db4f5fb6160dcc5fd57892(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:14:04,379 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 97532d2585300ca10ffc4f6d5dda82c5: 2024-12-04T19:14:04,379 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., storeName=97532d2585300ca10ffc4f6d5dda82c5/C, priority=13, startTime=1733339643841; duration=0sec 2024-12-04T19:14:04,379 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:14:04,379 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 97532d2585300ca10ffc4f6d5dda82c5:C 2024-12-04T19:14:04,425 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:04,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57632 deadline: 1733339704422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:04,425 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:04,425 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:04,425 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:04,431 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:04,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57600 deadline: 1733339704431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:04,432 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:04,432 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:04,432 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:04,432 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:04,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57588 deadline: 1733339704431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:04,433 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:04,433 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:04,433 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:04,433 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:04,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57662 deadline: 1733339704431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:04,434 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:04,434 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:04,434 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:04,434 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:04,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57646 deadline: 1733339704432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:04,434 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:04,434 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:04,434 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:04,436 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/B/ac6fe4b8706746e1974dfba27f3a69f6 2024-12-04T19:14:04,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/C/d8b911757c3e44fab3228269524eba8d is 50, key is test_row_0/C:col10/1733339642947/Put/seqid=0 2024-12-04T19:14:04,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741872_1048 (size=12301) 2024-12-04T19:14:04,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:04,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57632 deadline: 1733339704732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:04,734 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:04,734 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:04,734 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:04,744 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:04,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57600 deadline: 1733339704741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:04,745 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:04,745 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:04,745 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:04,745 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:04,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57588 deadline: 1733339704742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:04,745 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:04,745 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:04,745 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:04,745 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:04,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57662 deadline: 1733339704742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:04,746 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:04,746 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:04,746 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:04,746 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:04,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57646 deadline: 1733339704742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:04,747 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:04,747 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:04,747 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:04,855 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/C/d8b911757c3e44fab3228269524eba8d 2024-12-04T19:14:04,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/A/d4de660048744e9ba0352a0b0efdce22 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/d4de660048744e9ba0352a0b0efdce22 2024-12-04T19:14:04,872 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/d4de660048744e9ba0352a0b0efdce22, entries=150, sequenceid=351, filesize=12.0 K 2024-12-04T19:14:04,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/B/ac6fe4b8706746e1974dfba27f3a69f6 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/ac6fe4b8706746e1974dfba27f3a69f6 2024-12-04T19:14:04,880 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/ac6fe4b8706746e1974dfba27f3a69f6, entries=150, sequenceid=351, filesize=12.0 K 2024-12-04T19:14:04,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/C/d8b911757c3e44fab3228269524eba8d as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/d8b911757c3e44fab3228269524eba8d 2024-12-04T19:14:04,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-04T19:14:04,888 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/d8b911757c3e44fab3228269524eba8d, entries=150, sequenceid=351, filesize=12.0 K 2024-12-04T19:14:04,889 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for 97532d2585300ca10ffc4f6d5dda82c5 in 912ms, sequenceid=351, compaction requested=false 2024-12-04T19:14:04,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 97532d2585300ca10ffc4f6d5dda82c5: 2024-12-04T19:14:04,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:04,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-12-04T19:14:04,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-12-04T19:14:04,894 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-12-04T19:14:04,894 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1520 sec 2024-12-04T19:14:04,897 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 2.1620 sec 2024-12-04T19:14:05,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(8855): Flush requested on 97532d2585300ca10ffc4f6d5dda82c5 2024-12-04T19:14:05,244 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 97532d2585300ca10ffc4f6d5dda82c5 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-04T19:14:05,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97532d2585300ca10ffc4f6d5dda82c5, store=A 2024-12-04T19:14:05,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:14:05,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97532d2585300ca10ffc4f6d5dda82c5, store=B 2024-12-04T19:14:05,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:14:05,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97532d2585300ca10ffc4f6d5dda82c5, store=C 2024-12-04T19:14:05,245 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:14:05,266 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/A/6478e3ec41bd4f359a507437ee552614 is 50, key is test_row_0/A:col10/1733339644103/Put/seqid=0 2024-12-04T19:14:05,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741873_1049 (size=12301) 2024-12-04T19:14:05,280 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/A/6478e3ec41bd4f359a507437ee552614 2024-12-04T19:14:05,281 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733339645264, max timestamp=1733339645278Num uniques 859; 2024-12-04T19:14:05,281 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733339645264, max timestamp=1733339645278Num uniques 859; 2024-12-04T19:14:05,281 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-04T19:14:05,281 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733339645264, max timestamp=1733339645278Num uniques 859; 2024-12-04T19:14:05,281 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-04T19:14:05,281 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-04T19:14:05,298 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/B/cb163233217f441684b6853c6005330b is 50, key is test_row_0/B:col10/1733339644103/Put/seqid=0 2024-12-04T19:14:05,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741874_1050 (size=12301) 2024-12-04T19:14:05,309 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:05,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57588 deadline: 1733339705304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:05,310 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:05,310 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:05,310 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:05,310 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/B/cb163233217f441684b6853c6005330b 2024-12-04T19:14:05,315 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:05,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57632 deadline: 1733339705306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:05,316 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:05,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57600 deadline: 1733339705307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:05,316 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:05,316 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:05,317 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:05,317 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:05,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57662 deadline: 1733339705310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:05,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:05,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57646 deadline: 1733339705310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:05,317 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:05,317 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:05,317 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:05,318 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:05,318 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:05,318 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:05,318 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:05,318 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:05,318 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:05,326 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/C/1aa0e4ddd0a8441cbc89c7cdc8e0b590 is 50, key is test_row_0/C:col10/1733339644103/Put/seqid=0 2024-12-04T19:14:05,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741875_1051 (size=12301) 2024-12-04T19:14:05,340 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/C/1aa0e4ddd0a8441cbc89c7cdc8e0b590 2024-12-04T19:14:05,350 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/A/6478e3ec41bd4f359a507437ee552614 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/6478e3ec41bd4f359a507437ee552614 2024-12-04T19:14:05,359 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/6478e3ec41bd4f359a507437ee552614, entries=150, sequenceid=364, filesize=12.0 K 2024-12-04T19:14:05,361 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/B/cb163233217f441684b6853c6005330b as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/cb163233217f441684b6853c6005330b 2024-12-04T19:14:05,367 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/cb163233217f441684b6853c6005330b, entries=150, sequenceid=364, filesize=12.0 K 2024-12-04T19:14:05,368 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/C/1aa0e4ddd0a8441cbc89c7cdc8e0b590 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/1aa0e4ddd0a8441cbc89c7cdc8e0b590 2024-12-04T19:14:05,376 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/1aa0e4ddd0a8441cbc89c7cdc8e0b590, entries=150, sequenceid=364, filesize=12.0 K 2024-12-04T19:14:05,377 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 97532d2585300ca10ffc4f6d5dda82c5 in 133ms, sequenceid=364, compaction requested=true 2024-12-04T19:14:05,377 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 97532d2585300ca10ffc4f6d5dda82c5: 2024-12-04T19:14:05,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 97532d2585300ca10ffc4f6d5dda82c5:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T19:14:05,378 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T19:14:05,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:14:05,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 97532d2585300ca10ffc4f6d5dda82c5:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T19:14:05,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:14:05,378 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T19:14:05,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 97532d2585300ca10ffc4f6d5dda82c5:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T19:14:05,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T19:14:05,380 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37245 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T19:14:05,380 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1541): 97532d2585300ca10ffc4f6d5dda82c5/A is initiating minor compaction (all files) 2024-12-04T19:14:05,380 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37245 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T19:14:05,380 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 97532d2585300ca10ffc4f6d5dda82c5/A in TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:05,380 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1541): 97532d2585300ca10ffc4f6d5dda82c5/B is initiating minor compaction (all files) 2024-12-04T19:14:05,381 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 97532d2585300ca10ffc4f6d5dda82c5/B in TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:05,381 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/172a40f262ed4b00a129eb0eccdb9270, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/d4de660048744e9ba0352a0b0efdce22, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/6478e3ec41bd4f359a507437ee552614] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp, totalSize=36.4 K 2024-12-04T19:14:05,381 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/9cf5c91478ed4473aa486d01eec612e3, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/ac6fe4b8706746e1974dfba27f3a69f6, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/cb163233217f441684b6853c6005330b] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp, totalSize=36.4 K 2024-12-04T19:14:05,381 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 172a40f262ed4b00a129eb0eccdb9270, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1733339642878 2024-12-04T19:14:05,381 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 9cf5c91478ed4473aa486d01eec612e3, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1733339642878 2024-12-04T19:14:05,382 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting d4de660048744e9ba0352a0b0efdce22, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1733339642937 2024-12-04T19:14:05,382 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting ac6fe4b8706746e1974dfba27f3a69f6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1733339642937 2024-12-04T19:14:05,383 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6478e3ec41bd4f359a507437ee552614, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1733339644098 2024-12-04T19:14:05,383 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting cb163233217f441684b6853c6005330b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1733339644098 2024-12-04T19:14:05,397 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 97532d2585300ca10ffc4f6d5dda82c5#B#compaction#40 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:14:05,398 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 97532d2585300ca10ffc4f6d5dda82c5#A#compaction#39 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:14:05,398 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/B/e80aead9744a4b3d9a5ec095205b13ba is 50, key is test_row_0/B:col10/1733339644103/Put/seqid=0 2024-12-04T19:14:05,398 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/A/d6c0763bcc63432c9edbc0f9863a9b01 is 50, key is test_row_0/A:col10/1733339644103/Put/seqid=0 2024-12-04T19:14:05,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741876_1052 (size=12745) 2024-12-04T19:14:05,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741877_1053 (size=12745) 2024-12-04T19:14:05,420 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/A/d6c0763bcc63432c9edbc0f9863a9b01 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/d6c0763bcc63432c9edbc0f9863a9b01 2024-12-04T19:14:05,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(8855): Flush requested on 97532d2585300ca10ffc4f6d5dda82c5 2024-12-04T19:14:05,424 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 97532d2585300ca10ffc4f6d5dda82c5 3/3 column families, dataSize=140.89 KB heapSize=395.53 KB 2024-12-04T19:14:05,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97532d2585300ca10ffc4f6d5dda82c5, store=A 2024-12-04T19:14:05,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T19:14:05,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97532d2585300ca10ffc4f6d5dda82c5, store=B 2024-12-04T19:14:05,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T19:14:05,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97532d2585300ca10ffc4f6d5dda82c5, store=C 2024-12-04T19:14:05,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T19:14:05,434 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 97532d2585300ca10ffc4f6d5dda82c5/A of 97532d2585300ca10ffc4f6d5dda82c5 into d6c0763bcc63432c9edbc0f9863a9b01(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:14:05,434 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 97532d2585300ca10ffc4f6d5dda82c5: 2024-12-04T19:14:05,434 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., storeName=97532d2585300ca10ffc4f6d5dda82c5/A, priority=13, startTime=1733339645377; duration=0sec 2024-12-04T19:14:05,434 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T19:14:05,434 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 97532d2585300ca10ffc4f6d5dda82c5:A 2024-12-04T19:14:05,434 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T19:14:05,436 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37245 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T19:14:05,436 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1541): 97532d2585300ca10ffc4f6d5dda82c5/C is initiating minor compaction (all files) 2024-12-04T19:14:05,436 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 97532d2585300ca10ffc4f6d5dda82c5/C in TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:05,436 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/262f7d96d0db4f5fb6160dcc5fd57892, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/d8b911757c3e44fab3228269524eba8d, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/1aa0e4ddd0a8441cbc89c7cdc8e0b590] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp, totalSize=36.4 K 2024-12-04T19:14:05,437 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:05,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57632 deadline: 1733339705428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:05,438 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 262f7d96d0db4f5fb6160dcc5fd57892, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1733339642878 2024-12-04T19:14:05,438 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:05,438 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:05,438 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:05,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57588 deadline: 1733339705429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:05,438 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:05,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57662 deadline: 1733339705430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:05,438 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:05,438 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting d8b911757c3e44fab3228269524eba8d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1733339642937 2024-12-04T19:14:05,438 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:05,439 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:05,439 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:05,439 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1aa0e4ddd0a8441cbc89c7cdc8e0b590, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1733339644098 2024-12-04T19:14:05,440 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:05,440 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:05,440 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:05,444 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:05,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57600 deadline: 1733339705438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:05,445 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:05,445 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:05,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57646 deadline: 1733339705438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:05,445 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:05,445 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:05,445 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:05,445 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:05,445 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:05,453 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 97532d2585300ca10ffc4f6d5dda82c5#C#compaction#42 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:14:05,453 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/A/76684f10c0fa46c8b8b51579c8fc26e0 is 50, key is test_row_0/A:col10/1733339645295/Put/seqid=0 2024-12-04T19:14:05,454 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/C/a247866f99ac4593926f9431f157cee9 is 50, key is test_row_0/C:col10/1733339644103/Put/seqid=0 2024-12-04T19:14:05,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741878_1054 (size=14741) 2024-12-04T19:14:05,474 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=401 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/A/76684f10c0fa46c8b8b51579c8fc26e0 2024-12-04T19:14:05,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741879_1055 (size=12745) 2024-12-04T19:14:05,494 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/B/98c45a8d3ab044b1840e3ba40f8fd1bf is 50, key is test_row_0/B:col10/1733339645295/Put/seqid=0 2024-12-04T19:14:05,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741880_1056 (size=12301) 2024-12-04T19:14:05,508 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=401 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/B/98c45a8d3ab044b1840e3ba40f8fd1bf 2024-12-04T19:14:05,531 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/C/61ad7fe738b848e7b04804dfcbe9e9cb is 50, key is test_row_0/C:col10/1733339645295/Put/seqid=0 2024-12-04T19:14:05,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741881_1057 (size=12301) 2024-12-04T19:14:05,545 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=401 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/C/61ad7fe738b848e7b04804dfcbe9e9cb 2024-12-04T19:14:05,546 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:05,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57588 deadline: 1733339705543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:05,547 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:05,547 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:05,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57632 deadline: 1733339705543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:05,547 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:05,547 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:05,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:05,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:05,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:05,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:05,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57662 deadline: 1733339705552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:05,557 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:05,557 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:05,557 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:05,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57600 deadline: 1733339705553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:05,557 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:05,557 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:05,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57646 deadline: 1733339705553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:05,557 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:05,558 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:05,558 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:05,558 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:05,558 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:05,558 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:05,561 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/A/76684f10c0fa46c8b8b51579c8fc26e0 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/76684f10c0fa46c8b8b51579c8fc26e0 2024-12-04T19:14:05,569 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/76684f10c0fa46c8b8b51579c8fc26e0, entries=200, sequenceid=401, filesize=14.4 K 2024-12-04T19:14:05,572 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/B/98c45a8d3ab044b1840e3ba40f8fd1bf as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/98c45a8d3ab044b1840e3ba40f8fd1bf 2024-12-04T19:14:05,580 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/98c45a8d3ab044b1840e3ba40f8fd1bf, entries=150, sequenceid=401, filesize=12.0 K 2024-12-04T19:14:05,582 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/C/61ad7fe738b848e7b04804dfcbe9e9cb as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/61ad7fe738b848e7b04804dfcbe9e9cb 2024-12-04T19:14:05,593 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/61ad7fe738b848e7b04804dfcbe9e9cb, entries=150, sequenceid=401, filesize=12.0 K 2024-12-04T19:14:05,594 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~415.41 KB/425376, currentSize=33.54 KB/34350 for 97532d2585300ca10ffc4f6d5dda82c5 in 170ms, sequenceid=401, compaction requested=false 2024-12-04T19:14:05,595 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 97532d2585300ca10ffc4f6d5dda82c5: 2024-12-04T19:14:05,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(8855): Flush requested on 97532d2585300ca10ffc4f6d5dda82c5 2024-12-04T19:14:05,756 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 97532d2585300ca10ffc4f6d5dda82c5 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-04T19:14:05,756 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97532d2585300ca10ffc4f6d5dda82c5, store=A 2024-12-04T19:14:05,756 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:14:05,756 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97532d2585300ca10ffc4f6d5dda82c5, store=B 2024-12-04T19:14:05,757 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:14:05,757 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97532d2585300ca10ffc4f6d5dda82c5, store=C 2024-12-04T19:14:05,757 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:14:05,771 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/A/f83dce9ccdb94cfcb9fc06b219ebd06b is 50, key is test_row_0/A:col10/1733339645754/Put/seqid=0 2024-12-04T19:14:05,794 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733339645778, max timestamp=1733339645791Num uniques 859; 2024-12-04T19:14:05,794 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733339645778, max timestamp=1733339645791Num uniques 859; 2024-12-04T19:14:05,794 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T19:14:05,794 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T19:14:05,796 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733339645778, max timestamp=1733339645791Num uniques 859; 2024-12-04T19:14:05,796 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T19:14:05,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741882_1058 (size=12301) 2024-12-04T19:14:05,800 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=412 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/A/f83dce9ccdb94cfcb9fc06b219ebd06b 2024-12-04T19:14:05,818 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/B/e80aead9744a4b3d9a5ec095205b13ba as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/e80aead9744a4b3d9a5ec095205b13ba 2024-12-04T19:14:05,819 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/B/030ec87f2c42467895d43ffc0aae16a9 is 50, key is test_row_0/B:col10/1733339645754/Put/seqid=0 2024-12-04T19:14:05,828 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:05,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57646 deadline: 1733339705820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:05,828 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:05,828 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:05,829 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:05,829 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:05,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57600 deadline: 1733339705824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:05,830 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:05,830 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:05,830 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:05,830 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:05,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57662 deadline: 1733339705825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:05,831 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:05,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:05,831 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:05,831 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:05,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57588 deadline: 1733339705825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:05,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:05,831 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:05,831 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:05,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57632 deadline: 1733339705827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:05,831 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:05,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:05,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:05,832 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 97532d2585300ca10ffc4f6d5dda82c5/B of 97532d2585300ca10ffc4f6d5dda82c5 into e80aead9744a4b3d9a5ec095205b13ba(size=12.4 K), total size for store is 24.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:14:05,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:05,832 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 97532d2585300ca10ffc4f6d5dda82c5: 2024-12-04T19:14:05,832 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., storeName=97532d2585300ca10ffc4f6d5dda82c5/B, priority=13, startTime=1733339645378; duration=0sec 2024-12-04T19:14:05,832 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:14:05,832 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 97532d2585300ca10ffc4f6d5dda82c5:B 2024-12-04T19:14:05,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741883_1059 (size=12301) 2024-12-04T19:14:05,844 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=412 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/B/030ec87f2c42467895d43ffc0aae16a9 2024-12-04T19:14:05,858 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/C/6dce60eb67004a6a8e2126776181ce17 is 50, key is test_row_0/C:col10/1733339645754/Put/seqid=0 2024-12-04T19:14:05,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741884_1060 (size=12301) 2024-12-04T19:14:05,878 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=412 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/C/6dce60eb67004a6a8e2126776181ce17 2024-12-04T19:14:05,902 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/C/a247866f99ac4593926f9431f157cee9 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/a247866f99ac4593926f9431f157cee9 2024-12-04T19:14:05,905 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/A/f83dce9ccdb94cfcb9fc06b219ebd06b as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/f83dce9ccdb94cfcb9fc06b219ebd06b 2024-12-04T19:14:05,917 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 97532d2585300ca10ffc4f6d5dda82c5/C of 97532d2585300ca10ffc4f6d5dda82c5 into a247866f99ac4593926f9431f157cee9(size=12.4 K), total size for store is 24.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:14:05,917 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 97532d2585300ca10ffc4f6d5dda82c5: 2024-12-04T19:14:05,917 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., storeName=97532d2585300ca10ffc4f6d5dda82c5/C, priority=13, startTime=1733339645378; duration=0sec 2024-12-04T19:14:05,918 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:14:05,918 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 97532d2585300ca10ffc4f6d5dda82c5:C 2024-12-04T19:14:05,922 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/f83dce9ccdb94cfcb9fc06b219ebd06b, entries=150, sequenceid=412, filesize=12.0 K 2024-12-04T19:14:05,923 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/B/030ec87f2c42467895d43ffc0aae16a9 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/030ec87f2c42467895d43ffc0aae16a9 2024-12-04T19:14:05,931 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/030ec87f2c42467895d43ffc0aae16a9, entries=150, sequenceid=412, filesize=12.0 K 2024-12-04T19:14:05,933 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/C/6dce60eb67004a6a8e2126776181ce17 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/6dce60eb67004a6a8e2126776181ce17 2024-12-04T19:14:05,936 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:05,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57646 deadline: 1733339705932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:05,937 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:05,937 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:05,937 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:05,940 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/6dce60eb67004a6a8e2126776181ce17, entries=150, sequenceid=412, filesize=12.0 K 2024-12-04T19:14:05,942 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 97532d2585300ca10ffc4f6d5dda82c5 in 186ms, sequenceid=412, compaction requested=true 2024-12-04T19:14:05,942 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 97532d2585300ca10ffc4f6d5dda82c5: 2024-12-04T19:14:05,943 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T19:14:05,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 97532d2585300ca10ffc4f6d5dda82c5:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T19:14:05,944 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:14:05,945 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T19:14:05,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 97532d2585300ca10ffc4f6d5dda82c5:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T19:14:05,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:14:05,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 97532d2585300ca10ffc4f6d5dda82c5:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T19:14:05,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(8855): Flush requested on 97532d2585300ca10ffc4f6d5dda82c5 2024-12-04T19:14:05,945 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39787 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T19:14:05,945 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1541): 97532d2585300ca10ffc4f6d5dda82c5/A is initiating minor compaction (all files) 2024-12-04T19:14:05,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T19:14:05,946 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 97532d2585300ca10ffc4f6d5dda82c5/A in TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:05,946 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/d6c0763bcc63432c9edbc0f9863a9b01, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/76684f10c0fa46c8b8b51579c8fc26e0, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/f83dce9ccdb94cfcb9fc06b219ebd06b] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp, totalSize=38.9 K 2024-12-04T19:14:05,946 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 97532d2585300ca10ffc4f6d5dda82c5 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-12-04T19:14:05,946 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37347 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T19:14:05,946 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting d6c0763bcc63432c9edbc0f9863a9b01, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1733339644098 2024-12-04T19:14:05,946 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1541): 97532d2585300ca10ffc4f6d5dda82c5/B is initiating minor compaction (all files) 2024-12-04T19:14:05,946 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 97532d2585300ca10ffc4f6d5dda82c5/B in TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:05,947 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/e80aead9744a4b3d9a5ec095205b13ba, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/98c45a8d3ab044b1840e3ba40f8fd1bf, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/030ec87f2c42467895d43ffc0aae16a9] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp, totalSize=36.5 K 2024-12-04T19:14:05,947 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 76684f10c0fa46c8b8b51579c8fc26e0, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=401, earliestPutTs=1733339645295 2024-12-04T19:14:05,947 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97532d2585300ca10ffc4f6d5dda82c5, store=A 2024-12-04T19:14:05,947 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting f83dce9ccdb94cfcb9fc06b219ebd06b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1733339645433 2024-12-04T19:14:05,947 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting e80aead9744a4b3d9a5ec095205b13ba, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1733339644098 2024-12-04T19:14:05,947 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T19:14:05,948 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97532d2585300ca10ffc4f6d5dda82c5, store=B 2024-12-04T19:14:05,948 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T19:14:05,948 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97532d2585300ca10ffc4f6d5dda82c5, store=C 2024-12-04T19:14:05,948 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T19:14:05,948 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 98c45a8d3ab044b1840e3ba40f8fd1bf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=401, earliestPutTs=1733339645295 2024-12-04T19:14:05,949 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 030ec87f2c42467895d43ffc0aae16a9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1733339645433 2024-12-04T19:14:05,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:05,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57588 deadline: 1733339705957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:05,961 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:05,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57662 deadline: 1733339705958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:05,961 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:05,961 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:05,961 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:05,961 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:05,961 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:05,961 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:05,973 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:05,973 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:05,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57632 deadline: 1733339705960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:05,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57600 deadline: 1733339705962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:05,974 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:05,974 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:05,974 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:05,975 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:05,975 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:05,975 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:05,978 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 97532d2585300ca10ffc4f6d5dda82c5#A#compaction#49 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:14:05,979 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/A/aa86364401b543fbb21be4b3a8d8ccff is 50, key is test_row_0/A:col10/1733339645754/Put/seqid=0 2024-12-04T19:14:05,979 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/A/976ad7e1a2b5451bba298de00d8729e7 is 50, key is test_row_0/A:col10/1733339645821/Put/seqid=0 2024-12-04T19:14:05,982 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 97532d2585300ca10ffc4f6d5dda82c5#B#compaction#50 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:14:05,983 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/B/56a67a1194474da0b04d17914bdd6fac is 50, key is test_row_0/B:col10/1733339645754/Put/seqid=0 2024-12-04T19:14:06,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741885_1061 (size=14741) 2024-12-04T19:14:06,013 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=451 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/A/976ad7e1a2b5451bba298de00d8729e7 2024-12-04T19:14:06,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741886_1062 (size=12847) 2024-12-04T19:14:06,041 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/A/aa86364401b543fbb21be4b3a8d8ccff as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/aa86364401b543fbb21be4b3a8d8ccff 2024-12-04T19:14:06,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741887_1063 (size=12847) 2024-12-04T19:14:06,051 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/B/588a065d480541b2bb89161455f082a4 is 50, key is test_row_0/B:col10/1733339645821/Put/seqid=0 2024-12-04T19:14:06,052 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 97532d2585300ca10ffc4f6d5dda82c5/A of 97532d2585300ca10ffc4f6d5dda82c5 into aa86364401b543fbb21be4b3a8d8ccff(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:14:06,052 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 97532d2585300ca10ffc4f6d5dda82c5: 2024-12-04T19:14:06,052 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., storeName=97532d2585300ca10ffc4f6d5dda82c5/A, priority=13, startTime=1733339645942; duration=0sec 2024-12-04T19:14:06,053 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T19:14:06,053 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 97532d2585300ca10ffc4f6d5dda82c5:A 2024-12-04T19:14:06,053 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T19:14:06,055 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37347 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T19:14:06,055 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1541): 97532d2585300ca10ffc4f6d5dda82c5/C is initiating minor compaction (all files) 2024-12-04T19:14:06,055 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 97532d2585300ca10ffc4f6d5dda82c5/C in TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:06,055 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/a247866f99ac4593926f9431f157cee9, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/61ad7fe738b848e7b04804dfcbe9e9cb, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/6dce60eb67004a6a8e2126776181ce17] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp, totalSize=36.5 K 2024-12-04T19:14:06,056 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting a247866f99ac4593926f9431f157cee9, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1733339644098 2024-12-04T19:14:06,058 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 61ad7fe738b848e7b04804dfcbe9e9cb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=401, earliestPutTs=1733339645295 2024-12-04T19:14:06,058 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/B/56a67a1194474da0b04d17914bdd6fac as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/56a67a1194474da0b04d17914bdd6fac 2024-12-04T19:14:06,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741888_1064 (size=12301) 2024-12-04T19:14:06,060 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6dce60eb67004a6a8e2126776181ce17, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1733339645433 2024-12-04T19:14:06,060 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=451 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/B/588a065d480541b2bb89161455f082a4 2024-12-04T19:14:06,067 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 97532d2585300ca10ffc4f6d5dda82c5/B of 97532d2585300ca10ffc4f6d5dda82c5 into 56a67a1194474da0b04d17914bdd6fac(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:14:06,067 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 97532d2585300ca10ffc4f6d5dda82c5: 2024-12-04T19:14:06,067 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., storeName=97532d2585300ca10ffc4f6d5dda82c5/B, priority=13, startTime=1733339645944; duration=0sec 2024-12-04T19:14:06,067 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:14:06,068 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 97532d2585300ca10ffc4f6d5dda82c5:B 2024-12-04T19:14:06,073 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/C/e4d4745a4a0f4114837f5a7c89843c12 is 50, key is test_row_0/C:col10/1733339645821/Put/seqid=0 2024-12-04T19:14:06,080 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:06,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57588 deadline: 1733339706071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:06,081 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:06,081 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:06,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57662 deadline: 1733339706072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:06,081 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:06,081 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:06,082 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 97532d2585300ca10ffc4f6d5dda82c5#C#compaction#53 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:14:06,083 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/C/91b5c29479344ed4b73f314844174cea is 50, key is test_row_0/C:col10/1733339645754/Put/seqid=0 2024-12-04T19:14:06,083 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:06,083 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:06,083 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:06,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:06,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57632 deadline: 1733339706082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:06,086 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:06,086 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:06,086 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:06,086 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:06,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57600 deadline: 1733339706082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:06,087 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:06,087 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:06,087 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:06,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741889_1065 (size=12301) 2024-12-04T19:14:06,101 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=451 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/C/e4d4745a4a0f4114837f5a7c89843c12 2024-12-04T19:14:06,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741890_1066 (size=12847) 2024-12-04T19:14:06,113 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/A/976ad7e1a2b5451bba298de00d8729e7 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/976ad7e1a2b5451bba298de00d8729e7 2024-12-04T19:14:06,119 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/976ad7e1a2b5451bba298de00d8729e7, entries=200, sequenceid=451, filesize=14.4 K 2024-12-04T19:14:06,120 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/B/588a065d480541b2bb89161455f082a4 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/588a065d480541b2bb89161455f082a4 2024-12-04T19:14:06,126 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/588a065d480541b2bb89161455f082a4, entries=150, sequenceid=451, filesize=12.0 K 2024-12-04T19:14:06,128 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/C/e4d4745a4a0f4114837f5a7c89843c12 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/e4d4745a4a0f4114837f5a7c89843c12 2024-12-04T19:14:06,134 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/e4d4745a4a0f4114837f5a7c89843c12, entries=150, sequenceid=451, filesize=12.0 K 2024-12-04T19:14:06,135 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for 97532d2585300ca10ffc4f6d5dda82c5 in 189ms, sequenceid=451, compaction requested=false 2024-12-04T19:14:06,135 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 97532d2585300ca10ffc4f6d5dda82c5: 2024-12-04T19:14:06,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(8855): Flush requested on 97532d2585300ca10ffc4f6d5dda82c5 2024-12-04T19:14:06,144 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 97532d2585300ca10ffc4f6d5dda82c5 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-04T19:14:06,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97532d2585300ca10ffc4f6d5dda82c5, store=A 2024-12-04T19:14:06,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:14:06,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97532d2585300ca10ffc4f6d5dda82c5, store=B 2024-12-04T19:14:06,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:14:06,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97532d2585300ca10ffc4f6d5dda82c5, store=C 2024-12-04T19:14:06,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:14:06,153 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/A/1df5dc2349e94dfcafc73dd970f302be is 50, key is test_row_0/A:col10/1733339646143/Put/seqid=0 2024-12-04T19:14:06,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741891_1067 (size=14741) 2024-12-04T19:14:06,173 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=463 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/A/1df5dc2349e94dfcafc73dd970f302be 2024-12-04T19:14:06,195 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/B/dde2ff0bc8c1446293366320c362f21d is 50, key is test_row_0/B:col10/1733339646143/Put/seqid=0 2024-12-04T19:14:06,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741892_1068 (size=12301) 2024-12-04T19:14:06,208 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=463 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/B/dde2ff0bc8c1446293366320c362f21d 2024-12-04T19:14:06,222 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/C/f043e3a19a30454096ade83d61ecd7f3 is 50, key is test_row_0/C:col10/1733339646143/Put/seqid=0 2024-12-04T19:14:06,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741893_1069 (size=12301) 2024-12-04T19:14:06,248 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733339646205, max timestamp=1733339646245Num uniques 859; 2024-12-04T19:14:06,248 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733339646205, max timestamp=1733339646245Num uniques 859; 2024-12-04T19:14:06,248 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733339646205, max timestamp=1733339646245Num uniques 859; 2024-12-04T19:14:06,248 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T19:14:06,248 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T19:14:06,248 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T19:14:06,312 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:06,313 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:06,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57646 deadline: 1733339706300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:06,315 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733339646297, max timestamp=1733339646311Num uniques 1018; 2024-12-04T19:14:06,315 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733339646297, max timestamp=1733339646311Num uniques 1018; 2024-12-04T19:14:06,315 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-04T19:14:06,315 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-04T19:14:06,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57588 deadline: 1733339706300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:06,316 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:06,316 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:06,316 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:06,316 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:06,316 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:06,316 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:06,317 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733339646297, max timestamp=1733339646311Num uniques 1018; 2024-12-04T19:14:06,317 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-04T19:14:06,356 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:06,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57662 deadline: 1733339706349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:06,357 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:06,357 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:06,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57632 deadline: 1733339706350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:06,357 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:06,357 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:06,358 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:06,358 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:06,358 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:06,358 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:06,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57600 deadline: 1733339706352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:06,358 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:06,358 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:06,358 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:06,425 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:06,425 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:06,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57646 deadline: 1733339706422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:06,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57588 deadline: 1733339706422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:06,426 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:06,426 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:06,426 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:06,426 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:06,427 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:06,427 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:06,463 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:06,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57662 deadline: 1733339706462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:06,464 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:06,464 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:06,464 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:06,464 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:06,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57632 deadline: 1733339706462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:06,464 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:06,464 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:06,464 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:06,465 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:06,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57600 deadline: 1733339706462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:06,465 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:06,465 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:06,465 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:06,514 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/C/91b5c29479344ed4b73f314844174cea as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/91b5c29479344ed4b73f314844174cea 2024-12-04T19:14:06,529 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 97532d2585300ca10ffc4f6d5dda82c5/C of 97532d2585300ca10ffc4f6d5dda82c5 into 91b5c29479344ed4b73f314844174cea(size=12.5 K), total size for store is 24.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:14:06,529 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 97532d2585300ca10ffc4f6d5dda82c5: 2024-12-04T19:14:06,529 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., storeName=97532d2585300ca10ffc4f6d5dda82c5/C, priority=13, startTime=1733339645945; duration=0sec 2024-12-04T19:14:06,530 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:14:06,530 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 97532d2585300ca10ffc4f6d5dda82c5:C 2024-12-04T19:14:06,635 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:06,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57588 deadline: 1733339706631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:06,635 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:06,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57646 deadline: 1733339706632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:06,637 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:06,637 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:06,637 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=463 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/C/f043e3a19a30454096ade83d61ecd7f3 2024-12-04T19:14:06,637 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:06,641 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:06,641 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:06,641 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:06,652 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/A/1df5dc2349e94dfcafc73dd970f302be as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/1df5dc2349e94dfcafc73dd970f302be 2024-12-04T19:14:06,670 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/1df5dc2349e94dfcafc73dd970f302be, entries=200, sequenceid=463, filesize=14.4 K 2024-12-04T19:14:06,674 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:06,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57662 deadline: 1733339706672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:06,674 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:06,675 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:06,675 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:06,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57632 deadline: 1733339706673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:06,675 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:06,675 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:06,675 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:06,675 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:06,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:06,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57600 deadline: 1733339706677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:06,679 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:06,679 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:06,679 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:06,681 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/B/dde2ff0bc8c1446293366320c362f21d as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/dde2ff0bc8c1446293366320c362f21d 2024-12-04T19:14:06,686 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/dde2ff0bc8c1446293366320c362f21d, entries=150, sequenceid=463, filesize=12.0 K 2024-12-04T19:14:06,688 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/C/f043e3a19a30454096ade83d61ecd7f3 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/f043e3a19a30454096ade83d61ecd7f3 2024-12-04T19:14:06,697 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/f043e3a19a30454096ade83d61ecd7f3, entries=150, sequenceid=463, filesize=12.0 K 2024-12-04T19:14:06,701 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 97532d2585300ca10ffc4f6d5dda82c5 in 556ms, sequenceid=463, compaction requested=true 2024-12-04T19:14:06,701 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 97532d2585300ca10ffc4f6d5dda82c5: 2024-12-04T19:14:06,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 97532d2585300ca10ffc4f6d5dda82c5:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T19:14:06,701 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T19:14:06,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:14:06,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 97532d2585300ca10ffc4f6d5dda82c5:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T19:14:06,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:14:06,701 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T19:14:06,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 97532d2585300ca10ffc4f6d5dda82c5:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T19:14:06,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T19:14:06,704 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42329 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T19:14:06,704 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1541): 97532d2585300ca10ffc4f6d5dda82c5/A is initiating minor compaction (all files) 2024-12-04T19:14:06,704 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T19:14:06,704 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 97532d2585300ca10ffc4f6d5dda82c5/A in TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:06,704 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1541): 97532d2585300ca10ffc4f6d5dda82c5/B is initiating minor compaction (all files) 2024-12-04T19:14:06,704 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 97532d2585300ca10ffc4f6d5dda82c5/B in TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:06,704 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/aa86364401b543fbb21be4b3a8d8ccff, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/976ad7e1a2b5451bba298de00d8729e7, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/1df5dc2349e94dfcafc73dd970f302be] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp, totalSize=41.3 K 2024-12-04T19:14:06,704 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/56a67a1194474da0b04d17914bdd6fac, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/588a065d480541b2bb89161455f082a4, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/dde2ff0bc8c1446293366320c362f21d] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp, totalSize=36.6 K 2024-12-04T19:14:06,705 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa86364401b543fbb21be4b3a8d8ccff, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1733339645433 2024-12-04T19:14:06,705 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 56a67a1194474da0b04d17914bdd6fac, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1733339645433 2024-12-04T19:14:06,705 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 588a065d480541b2bb89161455f082a4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=451, earliestPutTs=1733339645821 2024-12-04T19:14:06,707 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 976ad7e1a2b5451bba298de00d8729e7, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=451, earliestPutTs=1733339645820 2024-12-04T19:14:06,707 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting dde2ff0bc8c1446293366320c362f21d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=463, earliestPutTs=1733339645954 2024-12-04T19:14:06,708 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1df5dc2349e94dfcafc73dd970f302be, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=463, earliestPutTs=1733339645954 2024-12-04T19:14:06,734 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 97532d2585300ca10ffc4f6d5dda82c5#B#compaction#57 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:14:06,735 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/B/9b90062db7c3409fb155cf8f517d9859 is 50, key is test_row_0/B:col10/1733339646143/Put/seqid=0 2024-12-04T19:14:06,741 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 97532d2585300ca10ffc4f6d5dda82c5#A#compaction#58 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:14:06,742 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/A/9521516ec8c5466497ff7a8000a00373 is 50, key is test_row_0/A:col10/1733339646143/Put/seqid=0 2024-12-04T19:14:06,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741895_1071 (size=12949) 2024-12-04T19:14:06,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741894_1070 (size=12949) 2024-12-04T19:14:06,816 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/A/9521516ec8c5466497ff7a8000a00373 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/9521516ec8c5466497ff7a8000a00373 2024-12-04T19:14:06,836 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 97532d2585300ca10ffc4f6d5dda82c5/A of 97532d2585300ca10ffc4f6d5dda82c5 into 9521516ec8c5466497ff7a8000a00373(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:14:06,837 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 97532d2585300ca10ffc4f6d5dda82c5: 2024-12-04T19:14:06,837 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., storeName=97532d2585300ca10ffc4f6d5dda82c5/A, priority=13, startTime=1733339646701; duration=0sec 2024-12-04T19:14:06,837 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T19:14:06,837 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 97532d2585300ca10ffc4f6d5dda82c5:A 2024-12-04T19:14:06,837 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T19:14:06,841 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T19:14:06,841 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1541): 97532d2585300ca10ffc4f6d5dda82c5/C is initiating minor compaction (all files) 2024-12-04T19:14:06,841 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 97532d2585300ca10ffc4f6d5dda82c5/C in TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:06,841 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/91b5c29479344ed4b73f314844174cea, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/e4d4745a4a0f4114837f5a7c89843c12, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/f043e3a19a30454096ade83d61ecd7f3] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp, totalSize=36.6 K 2024-12-04T19:14:06,842 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 91b5c29479344ed4b73f314844174cea, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1733339645433 2024-12-04T19:14:06,843 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting e4d4745a4a0f4114837f5a7c89843c12, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=451, earliestPutTs=1733339645821 2024-12-04T19:14:06,847 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting f043e3a19a30454096ade83d61ecd7f3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=463, earliestPutTs=1733339645954 2024-12-04T19:14:06,872 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 97532d2585300ca10ffc4f6d5dda82c5#C#compaction#59 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:14:06,873 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/C/318c88be542a4a7d9bb01773f39cb224 is 50, key is test_row_0/C:col10/1733339646143/Put/seqid=0 2024-12-04T19:14:06,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741896_1072 (size=12949) 2024-12-04T19:14:06,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-04T19:14:06,893 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T19:14:06,895 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T19:14:06,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-12-04T19:14:06,900 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T19:14:06,901 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T19:14:06,902 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T19:14:06,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-04T19:14:06,906 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/C/318c88be542a4a7d9bb01773f39cb224 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/318c88be542a4a7d9bb01773f39cb224 2024-12-04T19:14:06,922 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 97532d2585300ca10ffc4f6d5dda82c5/C of 97532d2585300ca10ffc4f6d5dda82c5 into 318c88be542a4a7d9bb01773f39cb224(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:14:06,922 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 97532d2585300ca10ffc4f6d5dda82c5: 2024-12-04T19:14:06,922 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., storeName=97532d2585300ca10ffc4f6d5dda82c5/C, priority=13, startTime=1733339646701; duration=0sec 2024-12-04T19:14:06,922 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:14:06,922 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 97532d2585300ca10ffc4f6d5dda82c5:C 2024-12-04T19:14:06,946 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 97532d2585300ca10ffc4f6d5dda82c5 3/3 column families, dataSize=140.89 KB heapSize=404.77 KB 2024-12-04T19:14:06,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97532d2585300ca10ffc4f6d5dda82c5, store=A 2024-12-04T19:14:06,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-04T19:14:06,947 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97532d2585300ca10ffc4f6d5dda82c5, store=B 2024-12-04T19:14:06,947 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-04T19:14:06,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(8855): Flush requested on 97532d2585300ca10ffc4f6d5dda82c5 2024-12-04T19:14:06,948 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97532d2585300ca10ffc4f6d5dda82c5, store=C 2024-12-04T19:14:06,948 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-04T19:14:06,959 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/A/6d0f240a5f3b41279b0532d9945f13c2 is 50, key is test_row_0/A:col10/1733339646348/Put/seqid=0 2024-12-04T19:14:06,974 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:06,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57588 deadline: 1733339706967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:06,975 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:06,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57646 deadline: 1733339706969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:06,976 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:06,976 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:06,976 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:06,980 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:06,980 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:06,980 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:06,987 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:06,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57662 deadline: 1733339706984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:06,987 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:06,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57632 deadline: 1733339706984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:06,988 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:06,988 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:06,988 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:06,988 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:06,988 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:06,988 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:06,993 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:06,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57600 deadline: 1733339706991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:06,994 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:06,994 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:06,994 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:07,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741897_1073 (size=14741) 2024-12-04T19:14:07,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-04T19:14:07,062 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-04T19:14:07,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:07,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. as already flushing 2024-12-04T19:14:07,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:07,063 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:07,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:07,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:07,085 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:07,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57646 deadline: 1733339707081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:07,086 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:07,086 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:07,086 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:07,086 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:07,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57588 deadline: 1733339707082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:07,087 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:07,087 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:07,087 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:07,205 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/B/9b90062db7c3409fb155cf8f517d9859 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/9b90062db7c3409fb155cf8f517d9859 2024-12-04T19:14:07,213 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 97532d2585300ca10ffc4f6d5dda82c5/B of 97532d2585300ca10ffc4f6d5dda82c5 into 9b90062db7c3409fb155cf8f517d9859(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:14:07,213 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 97532d2585300ca10ffc4f6d5dda82c5: 2024-12-04T19:14:07,213 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., storeName=97532d2585300ca10ffc4f6d5dda82c5/B, priority=13, startTime=1733339646701; duration=0sec 2024-12-04T19:14:07,213 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:14:07,214 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 97532d2585300ca10ffc4f6d5dda82c5:B 2024-12-04T19:14:07,217 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-04T19:14:07,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:07,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. as already flushing 2024-12-04T19:14:07,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:07,217 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:07,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:07,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:07,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-04T19:14:07,293 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:07,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57646 deadline: 1733339707291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:07,293 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:07,293 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:07,294 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:07,294 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:07,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57588 deadline: 1733339707291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:07,294 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:07,294 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:07,294 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:07,371 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-04T19:14:07,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:07,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. as already flushing 2024-12-04T19:14:07,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:07,372 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:07,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:07,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:07,407 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=521 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/A/6d0f240a5f3b41279b0532d9945f13c2 2024-12-04T19:14:07,416 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/B/118492520cfd4cd08019712385000d1f is 50, key is test_row_0/B:col10/1733339646348/Put/seqid=0 2024-12-04T19:14:07,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741898_1074 (size=12301) 2024-12-04T19:14:07,422 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=521 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/B/118492520cfd4cd08019712385000d1f 2024-12-04T19:14:07,431 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/C/1abf2b7267144301b7bb270b731cc19c is 50, key is test_row_0/C:col10/1733339646348/Put/seqid=0 2024-12-04T19:14:07,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741899_1075 (size=12301) 2024-12-04T19:14:07,503 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:07,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57662 deadline: 1733339707502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:07,504 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:07,504 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:07,504 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:07,504 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:07,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57632 deadline: 1733339707502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:07,505 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:07,505 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:07,505 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:07,505 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:07,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57600 deadline: 1733339707502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:07,506 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:07,506 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:07,506 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:07,524 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-04T19:14:07,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:07,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. as already flushing 2024-12-04T19:14:07,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:07,525 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:07,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:07,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:07,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-04T19:14:07,604 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:07,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57646 deadline: 1733339707601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:07,604 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:07,604 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:07,604 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:07,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57588 deadline: 1733339707601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:07,605 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:07,605 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:07,605 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:07,605 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:07,677 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-04T19:14:07,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:07,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. as already flushing 2024-12-04T19:14:07,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:07,678 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:07,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:07,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:07,830 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-04T19:14:07,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:07,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. as already flushing 2024-12-04T19:14:07,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:07,831 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:07,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:07,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:07,839 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=521 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/C/1abf2b7267144301b7bb270b731cc19c 2024-12-04T19:14:07,846 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/A/6d0f240a5f3b41279b0532d9945f13c2 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/6d0f240a5f3b41279b0532d9945f13c2 2024-12-04T19:14:07,851 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/6d0f240a5f3b41279b0532d9945f13c2, entries=200, sequenceid=521, filesize=14.4 K 2024-12-04T19:14:07,852 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/B/118492520cfd4cd08019712385000d1f as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/118492520cfd4cd08019712385000d1f 2024-12-04T19:14:07,857 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/118492520cfd4cd08019712385000d1f, entries=150, sequenceid=521, filesize=12.0 K 2024-12-04T19:14:07,859 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/C/1abf2b7267144301b7bb270b731cc19c as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/1abf2b7267144301b7bb270b731cc19c 2024-12-04T19:14:07,864 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/1abf2b7267144301b7bb270b731cc19c, entries=150, sequenceid=521, filesize=12.0 K 2024-12-04T19:14:07,865 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~404.72 KB/414432, currentSize=40.25 KB/41220 for 97532d2585300ca10ffc4f6d5dda82c5 in 919ms, sequenceid=521, compaction requested=false 2024-12-04T19:14:07,865 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 97532d2585300ca10ffc4f6d5dda82c5: 2024-12-04T19:14:07,983 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-04T19:14:07,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:07,984 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 97532d2585300ca10ffc4f6d5dda82c5 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-12-04T19:14:07,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97532d2585300ca10ffc4f6d5dda82c5, store=A 2024-12-04T19:14:07,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:14:07,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97532d2585300ca10ffc4f6d5dda82c5, store=B 2024-12-04T19:14:07,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:14:07,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97532d2585300ca10ffc4f6d5dda82c5, store=C 2024-12-04T19:14:07,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:14:07,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/A/8db49ed06754490b9ddca28e4cf8bdd9 is 50, key is test_row_0/A:col10/1733339646964/Put/seqid=0 2024-12-04T19:14:07,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741900_1076 (size=12301) 2024-12-04T19:14:07,995 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=531 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/A/8db49ed06754490b9ddca28e4cf8bdd9 2024-12-04T19:14:08,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/B/5fd32190041147ac98c8c06417907416 is 50, key is test_row_0/B:col10/1733339646964/Put/seqid=0 2024-12-04T19:14:08,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741901_1077 (size=12301) 2024-12-04T19:14:08,009 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=531 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/B/5fd32190041147ac98c8c06417907416 2024-12-04T19:14:08,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/C/910f817d1fe545588bd52367c5d1121d is 50, key is test_row_0/C:col10/1733339646964/Put/seqid=0 2024-12-04T19:14:08,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741902_1078 (size=12301) 2024-12-04T19:14:08,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-04T19:14:08,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(8855): Flush requested on 97532d2585300ca10ffc4f6d5dda82c5 2024-12-04T19:14:08,114 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. as already flushing 2024-12-04T19:14:08,154 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733339648134, max timestamp=1733339648151Num uniques 859; 2024-12-04T19:14:08,154 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T19:14:08,156 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733339648134, max timestamp=1733339648151Num uniques 859; 2024-12-04T19:14:08,156 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T19:14:08,158 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733339648134, max timestamp=1733339648151Num uniques 859; 2024-12-04T19:14:08,159 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T19:14:08,201 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733339648185, max timestamp=1733339648198Num uniques 1068; 2024-12-04T19:14:08,201 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733339648185, max timestamp=1733339648198Num uniques 1068; 2024-12-04T19:14:08,202 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-12-04T19:14:08,202 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733339648185, max timestamp=1733339648198Num uniques 1068; 2024-12-04T19:14:08,202 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-12-04T19:14:08,202 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-12-04T19:14:08,243 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:08,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57646 deadline: 1733339708241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:08,244 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:08,244 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:08,244 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:08,244 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:08,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57588 deadline: 1733339708243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:08,245 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:08,245 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:08,245 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:08,358 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:08,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57646 deadline: 1733339708352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:08,358 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:08,358 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:08,358 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:08,358 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:08,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57588 deadline: 1733339708352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:08,359 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:08,359 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:08,359 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:08,429 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=531 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/C/910f817d1fe545588bd52367c5d1121d 2024-12-04T19:14:08,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/A/8db49ed06754490b9ddca28e4cf8bdd9 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/8db49ed06754490b9ddca28e4cf8bdd9 2024-12-04T19:14:08,446 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/8db49ed06754490b9ddca28e4cf8bdd9, entries=150, sequenceid=531, filesize=12.0 K 2024-12-04T19:14:08,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/B/5fd32190041147ac98c8c06417907416 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/5fd32190041147ac98c8c06417907416 2024-12-04T19:14:08,454 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/5fd32190041147ac98c8c06417907416, entries=150, sequenceid=531, filesize=12.0 K 2024-12-04T19:14:08,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/C/910f817d1fe545588bd52367c5d1121d as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/910f817d1fe545588bd52367c5d1121d 2024-12-04T19:14:08,463 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/910f817d1fe545588bd52367c5d1121d, entries=150, sequenceid=531, filesize=12.0 K 2024-12-04T19:14:08,464 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for 97532d2585300ca10ffc4f6d5dda82c5 in 480ms, sequenceid=531, compaction requested=true 2024-12-04T19:14:08,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 97532d2585300ca10ffc4f6d5dda82c5: 2024-12-04T19:14:08,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:08,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-12-04T19:14:08,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-12-04T19:14:08,468 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-12-04T19:14:08,468 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5630 sec 2024-12-04T19:14:08,470 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 1.5720 sec 2024-12-04T19:14:08,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(8855): Flush requested on 97532d2585300ca10ffc4f6d5dda82c5 2024-12-04T19:14:08,513 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 97532d2585300ca10ffc4f6d5dda82c5 3/3 column families, dataSize=147.60 KB heapSize=420 KB 2024-12-04T19:14:08,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97532d2585300ca10ffc4f6d5dda82c5, store=A 2024-12-04T19:14:08,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-04T19:14:08,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97532d2585300ca10ffc4f6d5dda82c5, store=B 2024-12-04T19:14:08,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-04T19:14:08,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97532d2585300ca10ffc4f6d5dda82c5, store=C 2024-12-04T19:14:08,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-04T19:14:08,520 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/A/deaa1229f50743d6aa1fcbf8dce3e58b is 50, key is test_row_0/A:col10/1733339648231/Put/seqid=0 2024-12-04T19:14:08,523 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:08,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57632 deadline: 1733339708519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:08,524 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:08,524 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:08,524 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:08,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57662 deadline: 1733339708520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:08,524 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:08,524 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:08,524 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:08,524 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:08,525 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:08,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57600 deadline: 1733339708522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:08,528 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 9ba2734cd39f,33969,1733339629791 for 'test_row_1' in 97532d2585300ca10ffc4f6d5dda82c5 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-04T19:14:08,529 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:08,529 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:08,529 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:08,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741903_1079 (size=14741) 2024-12-04T19:14:08,534 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=586 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/A/deaa1229f50743d6aa1fcbf8dce3e58b 2024-12-04T19:14:08,546 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/B/c8a6ebbe8f83462cbac7d6a9ac7c7a0e is 50, key is test_row_0/B:col10/1733339648231/Put/seqid=0 2024-12-04T19:14:08,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741904_1080 (size=12301) 2024-12-04T19:14:08,558 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=586 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/B/c8a6ebbe8f83462cbac7d6a9ac7c7a0e 2024-12-04T19:14:08,565 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:08,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57646 deadline: 1733339708561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:08,565 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:08,565 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:08,566 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:08,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57588 deadline: 1733339708561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:08,566 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:08,566 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:08,566 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:08,566 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:08,566 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/C/a18415c5b0c0498190f230fac9cc0e80 is 50, key is test_row_0/C:col10/1733339648231/Put/seqid=0 2024-12-04T19:14:08,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741905_1081 (size=12301) 2024-12-04T19:14:08,577 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=586 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/C/a18415c5b0c0498190f230fac9cc0e80 2024-12-04T19:14:08,591 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/A/deaa1229f50743d6aa1fcbf8dce3e58b as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/deaa1229f50743d6aa1fcbf8dce3e58b 2024-12-04T19:14:08,599 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/deaa1229f50743d6aa1fcbf8dce3e58b, entries=200, sequenceid=586, filesize=14.4 K 2024-12-04T19:14:08,600 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/B/c8a6ebbe8f83462cbac7d6a9ac7c7a0e as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/c8a6ebbe8f83462cbac7d6a9ac7c7a0e 2024-12-04T19:14:08,605 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/c8a6ebbe8f83462cbac7d6a9ac7c7a0e, entries=150, sequenceid=586, filesize=12.0 K 2024-12-04T19:14:08,606 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/C/a18415c5b0c0498190f230fac9cc0e80 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/a18415c5b0c0498190f230fac9cc0e80 2024-12-04T19:14:08,613 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/a18415c5b0c0498190f230fac9cc0e80, entries=150, sequenceid=586, filesize=12.0 K 2024-12-04T19:14:08,614 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~419.95 KB/430032, currentSize=33.54 KB/34350 for 97532d2585300ca10ffc4f6d5dda82c5 in 101ms, sequenceid=586, compaction requested=true 2024-12-04T19:14:08,614 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 97532d2585300ca10ffc4f6d5dda82c5: 2024-12-04T19:14:08,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 97532d2585300ca10ffc4f6d5dda82c5:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T19:14:08,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:14:08,614 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T19:14:08,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 97532d2585300ca10ffc4f6d5dda82c5:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T19:14:08,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:14:08,614 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T19:14:08,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 97532d2585300ca10ffc4f6d5dda82c5:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T19:14:08,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T19:14:08,616 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54732 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T19:14:08,616 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T19:14:08,616 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1541): 97532d2585300ca10ffc4f6d5dda82c5/A is initiating minor compaction (all files) 2024-12-04T19:14:08,616 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1541): 97532d2585300ca10ffc4f6d5dda82c5/B is initiating minor compaction (all files) 2024-12-04T19:14:08,616 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 97532d2585300ca10ffc4f6d5dda82c5/A in TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:08,616 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 97532d2585300ca10ffc4f6d5dda82c5/B in TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:08,616 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/9521516ec8c5466497ff7a8000a00373, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/6d0f240a5f3b41279b0532d9945f13c2, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/8db49ed06754490b9ddca28e4cf8bdd9, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/deaa1229f50743d6aa1fcbf8dce3e58b] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp, totalSize=53.4 K 2024-12-04T19:14:08,616 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/9b90062db7c3409fb155cf8f517d9859, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/118492520cfd4cd08019712385000d1f, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/5fd32190041147ac98c8c06417907416, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/c8a6ebbe8f83462cbac7d6a9ac7c7a0e] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp, totalSize=48.7 K 2024-12-04T19:14:08,617 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9521516ec8c5466497ff7a8000a00373, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=463, earliestPutTs=1733339645954 2024-12-04T19:14:08,617 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b90062db7c3409fb155cf8f517d9859, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=463, earliestPutTs=1733339645954 2024-12-04T19:14:08,618 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 118492520cfd4cd08019712385000d1f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=521, earliestPutTs=1733339646348 2024-12-04T19:14:08,618 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6d0f240a5f3b41279b0532d9945f13c2, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=521, earliestPutTs=1733339646347 2024-12-04T19:14:08,618 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 5fd32190041147ac98c8c06417907416, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=531, earliestPutTs=1733339646963 2024-12-04T19:14:08,618 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8db49ed06754490b9ddca28e4cf8bdd9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=531, earliestPutTs=1733339646963 2024-12-04T19:14:08,618 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting deaa1229f50743d6aa1fcbf8dce3e58b, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=586, earliestPutTs=1733339648231 2024-12-04T19:14:08,618 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting c8a6ebbe8f83462cbac7d6a9ac7c7a0e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=586, earliestPutTs=1733339648231 2024-12-04T19:14:08,635 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 97532d2585300ca10ffc4f6d5dda82c5#A#compaction#69 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:14:08,635 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/A/8d8364f89bff449c84e8844a9596ea65 is 50, key is test_row_0/A:col10/1733339648231/Put/seqid=0 2024-12-04T19:14:08,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(8855): Flush requested on 97532d2585300ca10ffc4f6d5dda82c5 2024-12-04T19:14:08,638 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 97532d2585300ca10ffc4f6d5dda82c5 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-04T19:14:08,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97532d2585300ca10ffc4f6d5dda82c5, store=A 2024-12-04T19:14:08,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:14:08,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97532d2585300ca10ffc4f6d5dda82c5, store=B 2024-12-04T19:14:08,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:14:08,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97532d2585300ca10ffc4f6d5dda82c5, store=C 2024-12-04T19:14:08,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:14:08,648 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 97532d2585300ca10ffc4f6d5dda82c5#B#compaction#70 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:14:08,648 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/B/dc7b1da755c549dda6e5438320ce5f4a is 50, key is test_row_0/B:col10/1733339648231/Put/seqid=0 2024-12-04T19:14:08,650 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/A/3a9539b46678439a9b7e170071758c8e is 50, key is test_row_0/A:col10/1733339648517/Put/seqid=0 2024-12-04T19:14:08,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741906_1082 (size=13085) 2024-12-04T19:14:08,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741907_1083 (size=12301) 2024-12-04T19:14:08,677 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=596 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/A/3a9539b46678439a9b7e170071758c8e 2024-12-04T19:14:08,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741908_1084 (size=13085) 2024-12-04T19:14:08,701 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/B/dc7b1da755c549dda6e5438320ce5f4a as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/dc7b1da755c549dda6e5438320ce5f4a 2024-12-04T19:14:08,707 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/B/444446ab26da42e4a16a368b55b5b43b is 50, key is test_row_0/B:col10/1733339648517/Put/seqid=0 2024-12-04T19:14:08,710 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 97532d2585300ca10ffc4f6d5dda82c5/B of 97532d2585300ca10ffc4f6d5dda82c5 into dc7b1da755c549dda6e5438320ce5f4a(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:14:08,710 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 97532d2585300ca10ffc4f6d5dda82c5: 2024-12-04T19:14:08,710 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., storeName=97532d2585300ca10ffc4f6d5dda82c5/B, priority=12, startTime=1733339648614; duration=0sec 2024-12-04T19:14:08,710 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T19:14:08,710 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 97532d2585300ca10ffc4f6d5dda82c5:B 2024-12-04T19:14:08,711 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T19:14:08,713 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T19:14:08,713 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1541): 97532d2585300ca10ffc4f6d5dda82c5/C is initiating minor compaction (all files) 2024-12-04T19:14:08,714 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 97532d2585300ca10ffc4f6d5dda82c5/C in TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:08,714 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/318c88be542a4a7d9bb01773f39cb224, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/1abf2b7267144301b7bb270b731cc19c, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/910f817d1fe545588bd52367c5d1121d, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/a18415c5b0c0498190f230fac9cc0e80] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp, totalSize=48.7 K 2024-12-04T19:14:08,715 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 318c88be542a4a7d9bb01773f39cb224, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=463, earliestPutTs=1733339645954 2024-12-04T19:14:08,716 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 1abf2b7267144301b7bb270b731cc19c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=521, earliestPutTs=1733339646348 2024-12-04T19:14:08,716 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 910f817d1fe545588bd52367c5d1121d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=531, earliestPutTs=1733339646963 2024-12-04T19:14:08,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741909_1085 (size=12301) 2024-12-04T19:14:08,717 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting a18415c5b0c0498190f230fac9cc0e80, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=586, earliestPutTs=1733339648231 2024-12-04T19:14:08,718 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=596 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/B/444446ab26da42e4a16a368b55b5b43b 2024-12-04T19:14:08,723 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733339648678, max timestamp=1733339648720Num uniques 859; 2024-12-04T19:14:08,724 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T19:14:08,726 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733339648678, max timestamp=1733339648720Num uniques 859; 2024-12-04T19:14:08,726 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T19:14:08,728 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733339648678, max timestamp=1733339648720Num uniques 859; 2024-12-04T19:14:08,728 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T19:14:08,728 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/C/871a6b37337246948d8a18ea9a5699b8 is 50, key is test_row_0/C:col10/1733339648517/Put/seqid=0 2024-12-04T19:14:08,751 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 97532d2585300ca10ffc4f6d5dda82c5#C#compaction#74 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:14:08,752 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/C/a8bcc34de375484ebb76d81bc84cc21e is 50, key is test_row_0/C:col10/1733339648231/Put/seqid=0 2024-12-04T19:14:08,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741910_1086 (size=12301) 2024-12-04T19:14:08,799 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=596 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/C/871a6b37337246948d8a18ea9a5699b8 2024-12-04T19:14:08,803 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:08,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57662 deadline: 1733339708801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:08,804 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:08,804 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:08,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57632 deadline: 1733339708803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:08,804 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:08,805 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:08,805 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:08,805 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:08,805 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:08,806 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/A/3a9539b46678439a9b7e170071758c8e as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/3a9539b46678439a9b7e170071758c8e 2024-12-04T19:14:08,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741911_1087 (size=13085) 2024-12-04T19:14:08,814 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/3a9539b46678439a9b7e170071758c8e, entries=150, sequenceid=596, filesize=12.0 K 2024-12-04T19:14:08,816 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/B/444446ab26da42e4a16a368b55b5b43b as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/444446ab26da42e4a16a368b55b5b43b 2024-12-04T19:14:08,827 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/444446ab26da42e4a16a368b55b5b43b, entries=150, sequenceid=596, filesize=12.0 K 2024-12-04T19:14:08,831 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/C/871a6b37337246948d8a18ea9a5699b8 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/871a6b37337246948d8a18ea9a5699b8 2024-12-04T19:14:08,839 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/871a6b37337246948d8a18ea9a5699b8, entries=150, sequenceid=596, filesize=12.0 K 2024-12-04T19:14:08,840 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 97532d2585300ca10ffc4f6d5dda82c5 in 202ms, sequenceid=596, compaction requested=false 2024-12-04T19:14:08,840 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 97532d2585300ca10ffc4f6d5dda82c5: 2024-12-04T19:14:08,875 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 97532d2585300ca10ffc4f6d5dda82c5 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-12-04T19:14:08,875 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97532d2585300ca10ffc4f6d5dda82c5, store=A 2024-12-04T19:14:08,875 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-04T19:14:08,875 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97532d2585300ca10ffc4f6d5dda82c5, store=B 2024-12-04T19:14:08,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-04T19:14:08,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97532d2585300ca10ffc4f6d5dda82c5, store=C 2024-12-04T19:14:08,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-04T19:14:08,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(8855): Flush requested on 97532d2585300ca10ffc4f6d5dda82c5 2024-12-04T19:14:08,882 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/A/101112c1a7a142b4a6d27c5dd7df4dc0 is 50, key is test_row_0/A:col10/1733339648873/Put/seqid=0 2024-12-04T19:14:08,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741912_1088 (size=12301) 2024-12-04T19:14:08,893 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=635 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/A/101112c1a7a142b4a6d27c5dd7df4dc0 2024-12-04T19:14:08,902 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:08,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57588 deadline: 1733339708899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:08,903 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:08,903 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:08,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57646 deadline: 1733339708901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:08,903 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:08,903 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:08,904 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:08,904 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:08,904 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:08,914 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:08,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57662 deadline: 1733339708912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:08,915 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:08,915 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:08,915 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:08,915 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:08,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57632 deadline: 1733339708912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:08,916 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:08,916 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:08,916 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:08,921 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/B/e10369cb90964d01b7a37e9d83adc0d4 is 50, key is test_row_0/B:col10/1733339648873/Put/seqid=0 2024-12-04T19:14:08,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741913_1089 (size=12301) 2024-12-04T19:14:09,014 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:09,014 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:09,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57588 deadline: 1733339709012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:09,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57646 deadline: 1733339709012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:09,015 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:09,015 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:09,015 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:09,015 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:09,015 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:09,015 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:09,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-04T19:14:09,052 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T19:14:09,052 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T19:14:09,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-12-04T19:14:09,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-04T19:14:09,054 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T19:14:09,054 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T19:14:09,054 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T19:14:09,067 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/A/8d8364f89bff449c84e8844a9596ea65 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/8d8364f89bff449c84e8844a9596ea65 2024-12-04T19:14:09,076 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 97532d2585300ca10ffc4f6d5dda82c5/A of 97532d2585300ca10ffc4f6d5dda82c5 into 8d8364f89bff449c84e8844a9596ea65(size=12.8 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:14:09,076 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 97532d2585300ca10ffc4f6d5dda82c5: 2024-12-04T19:14:09,076 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., storeName=97532d2585300ca10ffc4f6d5dda82c5/A, priority=12, startTime=1733339648614; duration=0sec 2024-12-04T19:14:09,078 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:14:09,078 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 97532d2585300ca10ffc4f6d5dda82c5:A 2024-12-04T19:14:09,121 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:09,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57662 deadline: 1733339709121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:09,122 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:09,122 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:09,122 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:09,122 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:09,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57632 deadline: 1733339709121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:09,123 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:09,123 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:09,123 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:09,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-04T19:14:09,206 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-04T19:14:09,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:09,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. as already flushing 2024-12-04T19:14:09,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:09,207 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:09,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:09,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:09,217 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/C/a8bcc34de375484ebb76d81bc84cc21e as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/a8bcc34de375484ebb76d81bc84cc21e 2024-12-04T19:14:09,222 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:09,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57588 deadline: 1733339709222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:09,223 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:09,223 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 97532d2585300ca10ffc4f6d5dda82c5/C of 97532d2585300ca10ffc4f6d5dda82c5 into a8bcc34de375484ebb76d81bc84cc21e(size=12.8 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:14:09,223 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 97532d2585300ca10ffc4f6d5dda82c5: 2024-12-04T19:14:09,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57646 deadline: 1733339709222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:09,223 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., storeName=97532d2585300ca10ffc4f6d5dda82c5/C, priority=12, startTime=1733339648614; duration=0sec 2024-12-04T19:14:09,223 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:14:09,223 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 97532d2585300ca10ffc4f6d5dda82c5:C 2024-12-04T19:14:09,223 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:09,223 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:09,223 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:09,223 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:09,223 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:09,223 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:09,335 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=635 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/B/e10369cb90964d01b7a37e9d83adc0d4 2024-12-04T19:14:09,344 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/C/a761a6af9e9c4d2e9f373311bdfb7f2c is 50, key is test_row_0/C:col10/1733339648873/Put/seqid=0 2024-12-04T19:14:09,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741914_1090 (size=12301) 2024-12-04T19:14:09,353 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=635 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/C/a761a6af9e9c4d2e9f373311bdfb7f2c 2024-12-04T19:14:09,359 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/A/101112c1a7a142b4a6d27c5dd7df4dc0 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/101112c1a7a142b4a6d27c5dd7df4dc0 2024-12-04T19:14:09,359 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-04T19:14:09,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:09,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. as already flushing 2024-12-04T19:14:09,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:09,360 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:09,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:09,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:09,367 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/101112c1a7a142b4a6d27c5dd7df4dc0, entries=150, sequenceid=635, filesize=12.0 K 2024-12-04T19:14:09,368 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/B/e10369cb90964d01b7a37e9d83adc0d4 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/e10369cb90964d01b7a37e9d83adc0d4 2024-12-04T19:14:09,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-04T19:14:09,374 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/e10369cb90964d01b7a37e9d83adc0d4, entries=150, sequenceid=635, filesize=12.0 K 2024-12-04T19:14:09,375 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/C/a761a6af9e9c4d2e9f373311bdfb7f2c as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/a761a6af9e9c4d2e9f373311bdfb7f2c 2024-12-04T19:14:09,381 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/a761a6af9e9c4d2e9f373311bdfb7f2c, entries=150, sequenceid=635, filesize=12.0 K 2024-12-04T19:14:09,382 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~405.56 KB/415296, currentSize=40.25 KB/41220 for 97532d2585300ca10ffc4f6d5dda82c5 in 507ms, sequenceid=635, compaction requested=true 2024-12-04T19:14:09,382 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 97532d2585300ca10ffc4f6d5dda82c5: 2024-12-04T19:14:09,382 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 97532d2585300ca10ffc4f6d5dda82c5:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T19:14:09,382 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T19:14:09,382 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:14:09,382 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T19:14:09,382 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 97532d2585300ca10ffc4f6d5dda82c5:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T19:14:09,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:14:09,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 97532d2585300ca10ffc4f6d5dda82c5:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T19:14:09,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T19:14:09,384 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T19:14:09,384 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1541): 97532d2585300ca10ffc4f6d5dda82c5/B is initiating minor compaction (all files) 2024-12-04T19:14:09,384 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 97532d2585300ca10ffc4f6d5dda82c5/B in TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:09,385 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/dc7b1da755c549dda6e5438320ce5f4a, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/444446ab26da42e4a16a368b55b5b43b, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/e10369cb90964d01b7a37e9d83adc0d4] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp, totalSize=36.8 K 2024-12-04T19:14:09,385 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T19:14:09,385 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting dc7b1da755c549dda6e5438320ce5f4a, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=586, earliestPutTs=1733339648231 2024-12-04T19:14:09,385 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1541): 97532d2585300ca10ffc4f6d5dda82c5/A is initiating minor compaction (all files) 2024-12-04T19:14:09,385 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 97532d2585300ca10ffc4f6d5dda82c5/A in TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:09,385 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/8d8364f89bff449c84e8844a9596ea65, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/3a9539b46678439a9b7e170071758c8e, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/101112c1a7a142b4a6d27c5dd7df4dc0] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp, totalSize=36.8 K 2024-12-04T19:14:09,386 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 444446ab26da42e4a16a368b55b5b43b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=596, earliestPutTs=1733339648517 2024-12-04T19:14:09,387 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d8364f89bff449c84e8844a9596ea65, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=586, earliestPutTs=1733339648231 2024-12-04T19:14:09,387 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting e10369cb90964d01b7a37e9d83adc0d4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=635, earliestPutTs=1733339648797 2024-12-04T19:14:09,387 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3a9539b46678439a9b7e170071758c8e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=596, earliestPutTs=1733339648517 2024-12-04T19:14:09,388 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 101112c1a7a142b4a6d27c5dd7df4dc0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=635, earliestPutTs=1733339648797 2024-12-04T19:14:09,394 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 97532d2585300ca10ffc4f6d5dda82c5#A#compaction#78 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:14:09,395 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 97532d2585300ca10ffc4f6d5dda82c5#B#compaction#79 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:14:09,395 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/A/3bf21243c456499a86f4b4326c850ff1 is 50, key is test_row_0/A:col10/1733339648873/Put/seqid=0 2024-12-04T19:14:09,395 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/B/5129fb9f4cb74c648896c5ea30ea87de is 50, key is test_row_0/B:col10/1733339648873/Put/seqid=0 2024-12-04T19:14:09,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741915_1091 (size=13187) 2024-12-04T19:14:09,412 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/A/3bf21243c456499a86f4b4326c850ff1 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/3bf21243c456499a86f4b4326c850ff1 2024-12-04T19:14:09,420 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 97532d2585300ca10ffc4f6d5dda82c5/A of 97532d2585300ca10ffc4f6d5dda82c5 into 3bf21243c456499a86f4b4326c850ff1(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:14:09,420 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 97532d2585300ca10ffc4f6d5dda82c5: 2024-12-04T19:14:09,421 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., storeName=97532d2585300ca10ffc4f6d5dda82c5/A, priority=13, startTime=1733339649382; duration=0sec 2024-12-04T19:14:09,421 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T19:14:09,421 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 97532d2585300ca10ffc4f6d5dda82c5:A 2024-12-04T19:14:09,421 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T19:14:09,423 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T19:14:09,423 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1541): 97532d2585300ca10ffc4f6d5dda82c5/C is initiating minor compaction (all files) 2024-12-04T19:14:09,423 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 97532d2585300ca10ffc4f6d5dda82c5/C in TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:09,423 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/a8bcc34de375484ebb76d81bc84cc21e, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/871a6b37337246948d8a18ea9a5699b8, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/a761a6af9e9c4d2e9f373311bdfb7f2c] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp, totalSize=36.8 K 2024-12-04T19:14:09,424 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting a8bcc34de375484ebb76d81bc84cc21e, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=586, earliestPutTs=1733339648231 2024-12-04T19:14:09,424 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 871a6b37337246948d8a18ea9a5699b8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=596, earliestPutTs=1733339648517 2024-12-04T19:14:09,424 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting a761a6af9e9c4d2e9f373311bdfb7f2c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=635, earliestPutTs=1733339648797 2024-12-04T19:14:09,434 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 97532d2585300ca10ffc4f6d5dda82c5 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-04T19:14:09,434 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97532d2585300ca10ffc4f6d5dda82c5, store=A 2024-12-04T19:14:09,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:14:09,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97532d2585300ca10ffc4f6d5dda82c5, store=B 2024-12-04T19:14:09,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:14:09,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97532d2585300ca10ffc4f6d5dda82c5, store=C 2024-12-04T19:14:09,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:14:09,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(8855): Flush requested on 97532d2585300ca10ffc4f6d5dda82c5 2024-12-04T19:14:09,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741916_1092 (size=13187) 2024-12-04T19:14:09,457 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/B/5129fb9f4cb74c648896c5ea30ea87de as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/5129fb9f4cb74c648896c5ea30ea87de 2024-12-04T19:14:09,470 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 97532d2585300ca10ffc4f6d5dda82c5/B of 97532d2585300ca10ffc4f6d5dda82c5 into 5129fb9f4cb74c648896c5ea30ea87de(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:14:09,470 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 97532d2585300ca10ffc4f6d5dda82c5: 2024-12-04T19:14:09,470 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., storeName=97532d2585300ca10ffc4f6d5dda82c5/B, priority=13, startTime=1733339649382; duration=0sec 2024-12-04T19:14:09,471 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:14:09,471 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 97532d2585300ca10ffc4f6d5dda82c5:B 2024-12-04T19:14:09,477 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 97532d2585300ca10ffc4f6d5dda82c5#C#compaction#80 average throughput is 0.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:14:09,477 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/C/e7cbcacc77824f6588365730273267d3 is 50, key is test_row_0/C:col10/1733339648873/Put/seqid=0 2024-12-04T19:14:09,477 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/A/a9cce18da906429b8643be8d1ce1411f is 50, key is test_row_0/A:col10/1733339648881/Put/seqid=0 2024-12-04T19:14:09,482 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733339649470, max timestamp=1733339649480Num uniques 859; 2024-12-04T19:14:09,482 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T19:14:09,483 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733339649470, max timestamp=1733339649480Num uniques 859; 2024-12-04T19:14:09,483 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T19:14:09,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741917_1093 (size=13187) 2024-12-04T19:14:09,484 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733339649470, max timestamp=1733339649480Num uniques 859; 2024-12-04T19:14:09,484 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T19:14:09,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741918_1094 (size=12301) 2024-12-04T19:14:09,494 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=648 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/A/a9cce18da906429b8643be8d1ce1411f 2024-12-04T19:14:09,496 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/C/e7cbcacc77824f6588365730273267d3 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/e7cbcacc77824f6588365730273267d3 2024-12-04T19:14:09,508 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 97532d2585300ca10ffc4f6d5dda82c5/C of 97532d2585300ca10ffc4f6d5dda82c5 into e7cbcacc77824f6588365730273267d3(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:14:09,509 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 97532d2585300ca10ffc4f6d5dda82c5: 2024-12-04T19:14:09,509 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., storeName=97532d2585300ca10ffc4f6d5dda82c5/C, priority=13, startTime=1733339649383; duration=0sec 2024-12-04T19:14:09,509 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:14:09,509 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 97532d2585300ca10ffc4f6d5dda82c5:C 2024-12-04T19:14:09,513 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-04T19:14:09,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:09,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. as already flushing 2024-12-04T19:14:09,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:09,513 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/B/9596d44d850b4f2c95e0f6fa1042fb7f is 50, key is test_row_0/B:col10/1733339648881/Put/seqid=0 2024-12-04T19:14:09,513 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:09,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:09,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:09,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741919_1095 (size=12301) 2024-12-04T19:14:09,534 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=648 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/B/9596d44d850b4f2c95e0f6fa1042fb7f 2024-12-04T19:14:09,540 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:09,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57632 deadline: 1733339709535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:09,541 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:09,541 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:09,541 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:09,541 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:09,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57662 deadline: 1733339709538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:09,541 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:09,541 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:09,541 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:09,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57646 deadline: 1733339709540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:09,541 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:09,541 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:09,542 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:09,542 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:09,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57588 deadline: 1733339709540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:09,542 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:09,542 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:09,542 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:09,542 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:09,550 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/C/cfc54bad225644bd8208a72fc8a9ff9d is 50, key is test_row_0/C:col10/1733339648881/Put/seqid=0 2024-12-04T19:14:09,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741920_1096 (size=12301) 2024-12-04T19:14:09,556 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=648 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/C/cfc54bad225644bd8208a72fc8a9ff9d 2024-12-04T19:14:09,566 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/A/a9cce18da906429b8643be8d1ce1411f as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/a9cce18da906429b8643be8d1ce1411f 2024-12-04T19:14:09,578 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/a9cce18da906429b8643be8d1ce1411f, entries=150, sequenceid=648, filesize=12.0 K 2024-12-04T19:14:09,580 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/B/9596d44d850b4f2c95e0f6fa1042fb7f as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/9596d44d850b4f2c95e0f6fa1042fb7f 2024-12-04T19:14:09,588 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/9596d44d850b4f2c95e0f6fa1042fb7f, entries=150, sequenceid=648, filesize=12.0 K 2024-12-04T19:14:09,589 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/C/cfc54bad225644bd8208a72fc8a9ff9d as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/cfc54bad225644bd8208a72fc8a9ff9d 2024-12-04T19:14:09,598 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/cfc54bad225644bd8208a72fc8a9ff9d, entries=150, sequenceid=648, filesize=12.0 K 2024-12-04T19:14:09,599 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 97532d2585300ca10ffc4f6d5dda82c5 in 165ms, sequenceid=648, compaction requested=false 2024-12-04T19:14:09,599 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 97532d2585300ca10ffc4f6d5dda82c5: 2024-12-04T19:14:09,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(8855): Flush requested on 97532d2585300ca10ffc4f6d5dda82c5 2024-12-04T19:14:09,658 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 97532d2585300ca10ffc4f6d5dda82c5 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-12-04T19:14:09,659 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733339649532, max timestamp=1733339649656Num uniques 1018; 2024-12-04T19:14:09,659 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-04T19:14:09,661 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733339649532, max timestamp=1733339649656Num uniques 1018; 2024-12-04T19:14:09,661 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=6870, suffix heap size=133840,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=150 2024-12-04T19:14:09,662 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733339649532, max timestamp=1733339649656Num uniques 1018; 2024-12-04T19:14:09,662 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=6870, suffix heap size=133840,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=150 2024-12-04T19:14:09,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97532d2585300ca10ffc4f6d5dda82c5, store=A 2024-12-04T19:14:09,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T19:14:09,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97532d2585300ca10ffc4f6d5dda82c5, store=B 2024-12-04T19:14:09,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T19:14:09,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97532d2585300ca10ffc4f6d5dda82c5, store=C 2024-12-04T19:14:09,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T19:14:09,666 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-04T19:14:09,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:09,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. as already flushing 2024-12-04T19:14:09,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:09,667 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:09,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:09,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:09,669 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/A/9b6de1d4f1bd437e857134700f089e80 is 50, key is test_row_0/A:col10/1733339649657/Put/seqid=0 2024-12-04T19:14:09,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-04T19:14:09,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741921_1097 (size=12301) 2024-12-04T19:14:09,688 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=689 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/A/9b6de1d4f1bd437e857134700f089e80 2024-12-04T19:14:09,700 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/B/6a01c44df0534f7c8020da9ebaf4583b is 50, key is test_row_0/B:col10/1733339649657/Put/seqid=0 2024-12-04T19:14:09,700 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733339649671, max timestamp=1733339649697Num uniques 859; 2024-12-04T19:14:09,700 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-04T19:14:09,703 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733339649671, max timestamp=1733339649697Num uniques 859; 2024-12-04T19:14:09,703 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-04T19:14:09,704 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733339649671, max timestamp=1733339649697Num uniques 859; 2024-12-04T19:14:09,704 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-04T19:14:09,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741922_1098 (size=12301) 2024-12-04T19:14:09,711 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=10.38 KB at sequenceid=689 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/B/6a01c44df0534f7c8020da9ebaf4583b 2024-12-04T19:14:09,722 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/C/9cf126b8ba8142af96f62597a24537e0 is 50, key is test_row_0/C:col10/1733339649657/Put/seqid=0 2024-12-04T19:14:09,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741923_1099 (size=12301) 2024-12-04T19:14:09,740 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:09,740 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:09,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57632 deadline: 1733339709736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:09,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57662 deadline: 1733339709734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:09,741 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:09,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57588 deadline: 1733339709737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:09,741 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:09,741 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:09,741 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:09,741 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:09,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57646 deadline: 1733339709739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:09,741 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:09,741 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:09,742 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:09,742 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:09,742 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:09,742 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:09,742 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:09,742 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:09,742 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:09,821 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-04T19:14:09,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:09,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. as already flushing 2024-12-04T19:14:09,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:09,822 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:09,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:09,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:09,853 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:09,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57662 deadline: 1733339709852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:09,854 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:09,854 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:09,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57588 deadline: 1733339709852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:09,854 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:09,854 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:09,854 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:09,854 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:09,854 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:09,854 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:09,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57646 deadline: 1733339709852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:09,855 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:09,855 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:09,855 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:09,855 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:09,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57632 deadline: 1733339709852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:09,855 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:09,855 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:09,855 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:09,974 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-04T19:14:09,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:09,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. as already flushing 2024-12-04T19:14:09,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:09,975 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:09,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:09,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:10,062 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:10,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57662 deadline: 1733339710062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:10,063 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:10,063 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:10,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57646 deadline: 1733339710062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:10,063 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:10,063 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:10,063 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:10,063 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:10,063 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:10,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57588 deadline: 1733339710062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:10,063 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:10,064 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:10,064 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:10,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57632 deadline: 1733339710062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:10,064 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:10,064 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:10,064 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:10,064 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:10,064 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:10,128 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-04T19:14:10,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:10,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. as already flushing 2024-12-04T19:14:10,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:10,128 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:10,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:10,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:10,137 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=10.38 KB at sequenceid=689 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/C/9cf126b8ba8142af96f62597a24537e0 2024-12-04T19:14:10,151 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/A/9b6de1d4f1bd437e857134700f089e80 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/9b6de1d4f1bd437e857134700f089e80 2024-12-04T19:14:10,159 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/9b6de1d4f1bd437e857134700f089e80, entries=150, sequenceid=689, filesize=12.0 K 2024-12-04T19:14:10,161 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/B/6a01c44df0534f7c8020da9ebaf4583b as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/6a01c44df0534f7c8020da9ebaf4583b 2024-12-04T19:14:10,169 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/6a01c44df0534f7c8020da9ebaf4583b, entries=150, sequenceid=689, filesize=12.0 K 2024-12-04T19:14:10,171 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/C/9cf126b8ba8142af96f62597a24537e0 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/9cf126b8ba8142af96f62597a24537e0 2024-12-04T19:14:10,180 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/9cf126b8ba8142af96f62597a24537e0, entries=150, sequenceid=689, filesize=12.0 K 2024-12-04T19:14:10,181 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~31.95 KB/32714, heapSize ~81.61 KB/83568, currentSize=154.31 KB/158010 for 97532d2585300ca10ffc4f6d5dda82c5 in 524ms, sequenceid=689, compaction requested=true 2024-12-04T19:14:10,182 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 97532d2585300ca10ffc4f6d5dda82c5: 2024-12-04T19:14:10,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 97532d2585300ca10ffc4f6d5dda82c5:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T19:14:10,182 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T19:14:10,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:14:10,182 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T19:14:10,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 97532d2585300ca10ffc4f6d5dda82c5:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T19:14:10,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:14:10,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 97532d2585300ca10ffc4f6d5dda82c5:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T19:14:10,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T19:14:10,183 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T19:14:10,183 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1541): 97532d2585300ca10ffc4f6d5dda82c5/A is initiating minor compaction (all files) 2024-12-04T19:14:10,183 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T19:14:10,183 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 97532d2585300ca10ffc4f6d5dda82c5/A in TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:10,183 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1541): 97532d2585300ca10ffc4f6d5dda82c5/B is initiating minor compaction (all files) 2024-12-04T19:14:10,183 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 97532d2585300ca10ffc4f6d5dda82c5/B in TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:10,183 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/3bf21243c456499a86f4b4326c850ff1, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/a9cce18da906429b8643be8d1ce1411f, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/9b6de1d4f1bd437e857134700f089e80] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp, totalSize=36.9 K 2024-12-04T19:14:10,184 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/5129fb9f4cb74c648896c5ea30ea87de, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/9596d44d850b4f2c95e0f6fa1042fb7f, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/6a01c44df0534f7c8020da9ebaf4583b] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp, totalSize=36.9 K 2024-12-04T19:14:10,184 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3bf21243c456499a86f4b4326c850ff1, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=635, earliestPutTs=1733339648797 2024-12-04T19:14:10,184 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 5129fb9f4cb74c648896c5ea30ea87de, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=635, earliestPutTs=1733339648797 2024-12-04T19:14:10,184 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting a9cce18da906429b8643be8d1ce1411f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=648, earliestPutTs=1733339648881 2024-12-04T19:14:10,184 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 9596d44d850b4f2c95e0f6fa1042fb7f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=648, earliestPutTs=1733339648881 2024-12-04T19:14:10,185 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b6de1d4f1bd437e857134700f089e80, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=689, earliestPutTs=1733339649532 2024-12-04T19:14:10,185 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a01c44df0534f7c8020da9ebaf4583b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=689, earliestPutTs=1733339649532 2024-12-04T19:14:10,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-04T19:14:10,203 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 97532d2585300ca10ffc4f6d5dda82c5#B#compaction#87 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:14:10,204 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/B/7269dc6260804590b7fedc61b386ae9e is 50, key is test_row_0/B:col10/1733339649657/Put/seqid=0 2024-12-04T19:14:10,207 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 97532d2585300ca10ffc4f6d5dda82c5#A#compaction#88 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:14:10,207 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/A/ca95ba3aa611478cb4b5b6d185dbc125 is 50, key is test_row_0/A:col10/1733339649657/Put/seqid=0 2024-12-04T19:14:10,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741924_1100 (size=13289) 2024-12-04T19:14:10,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741925_1101 (size=13289) 2024-12-04T19:14:10,240 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/A/ca95ba3aa611478cb4b5b6d185dbc125 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/ca95ba3aa611478cb4b5b6d185dbc125 2024-12-04T19:14:10,248 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 97532d2585300ca10ffc4f6d5dda82c5/A of 97532d2585300ca10ffc4f6d5dda82c5 into ca95ba3aa611478cb4b5b6d185dbc125(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:14:10,249 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 97532d2585300ca10ffc4f6d5dda82c5: 2024-12-04T19:14:10,249 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., storeName=97532d2585300ca10ffc4f6d5dda82c5/A, priority=13, startTime=1733339650182; duration=0sec 2024-12-04T19:14:10,249 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T19:14:10,249 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 97532d2585300ca10ffc4f6d5dda82c5:A 2024-12-04T19:14:10,249 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T19:14:10,253 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T19:14:10,253 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1541): 97532d2585300ca10ffc4f6d5dda82c5/C is initiating minor compaction (all files) 2024-12-04T19:14:10,254 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 97532d2585300ca10ffc4f6d5dda82c5/C in TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:10,254 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/e7cbcacc77824f6588365730273267d3, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/cfc54bad225644bd8208a72fc8a9ff9d, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/9cf126b8ba8142af96f62597a24537e0] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp, totalSize=36.9 K 2024-12-04T19:14:10,254 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting e7cbcacc77824f6588365730273267d3, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=635, earliestPutTs=1733339648797 2024-12-04T19:14:10,255 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting cfc54bad225644bd8208a72fc8a9ff9d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=648, earliestPutTs=1733339648881 2024-12-04T19:14:10,256 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9cf126b8ba8142af96f62597a24537e0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=689, earliestPutTs=1733339649532 2024-12-04T19:14:10,274 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 97532d2585300ca10ffc4f6d5dda82c5#C#compaction#89 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:14:10,275 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/C/42e2fc4952e74da5b14fb810f5f46417 is 50, key is test_row_0/C:col10/1733339649657/Put/seqid=0 2024-12-04T19:14:10,281 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-04T19:14:10,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:10,281 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 97532d2585300ca10ffc4f6d5dda82c5 3/3 column families, dataSize=154.31 KB heapSize=430.69 KB 2024-12-04T19:14:10,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97532d2585300ca10ffc4f6d5dda82c5, store=A 2024-12-04T19:14:10,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T19:14:10,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97532d2585300ca10ffc4f6d5dda82c5, store=B 2024-12-04T19:14:10,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T19:14:10,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97532d2585300ca10ffc4f6d5dda82c5, store=C 2024-12-04T19:14:10,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T19:14:10,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741926_1102 (size=13289) 2024-12-04T19:14:10,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/A/dadc7fdfe9204f92897e903c250815e0 is 50, key is test_row_0/A:col10/1733339649738/Put/seqid=0 2024-12-04T19:14:10,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741927_1103 (size=12301) 2024-12-04T19:14:10,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(8855): Flush requested on 97532d2585300ca10ffc4f6d5dda82c5 2024-12-04T19:14:10,374 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. as already flushing 2024-12-04T19:14:10,378 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:10,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57662 deadline: 1733339710375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:10,378 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:10,379 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:10,379 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:10,382 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:10,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57588 deadline: 1733339710377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:10,383 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:10,382 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:10,383 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:10,383 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:10,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57646 deadline: 1733339710377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:10,383 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:10,383 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:10,383 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:10,383 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:10,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57632 deadline: 1733339710378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:10,383 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:10,383 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:10,383 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:10,486 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:10,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57662 deadline: 1733339710481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:10,487 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:10,487 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:10,487 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:10,494 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:10,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57588 deadline: 1733339710492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:10,495 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:10,495 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:10,495 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:10,495 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:10,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57646 deadline: 1733339710492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:10,495 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:10,495 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:10,495 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:10,495 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:10,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57632 deadline: 1733339710492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:10,496 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:10,496 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:10,496 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:10,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:10,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57600 deadline: 1733339710551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:10,556 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 9ba2734cd39f,33969,1733339629791 for 'test_row_1' in 97532d2585300ca10ffc4f6d5dda82c5 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-04T19:14:10,557 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:10,557 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:10,557 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:10,630 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/B/7269dc6260804590b7fedc61b386ae9e as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/7269dc6260804590b7fedc61b386ae9e 2024-12-04T19:14:10,637 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 97532d2585300ca10ffc4f6d5dda82c5/B of 97532d2585300ca10ffc4f6d5dda82c5 into 7269dc6260804590b7fedc61b386ae9e(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:14:10,637 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 97532d2585300ca10ffc4f6d5dda82c5: 2024-12-04T19:14:10,637 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., storeName=97532d2585300ca10ffc4f6d5dda82c5/B, priority=13, startTime=1733339650182; duration=0sec 2024-12-04T19:14:10,637 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:14:10,637 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 97532d2585300ca10ffc4f6d5dda82c5:B 2024-12-04T19:14:10,694 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:10,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57662 deadline: 1733339710692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:10,696 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:10,696 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:10,696 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:10,701 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/C/42e2fc4952e74da5b14fb810f5f46417 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/42e2fc4952e74da5b14fb810f5f46417 2024-12-04T19:14:10,705 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:10,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57588 deadline: 1733339710702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:10,706 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:10,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57646 deadline: 1733339710702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:10,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:10,706 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:10,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:10,707 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:10,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57632 deadline: 1733339710702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:10,707 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:10,707 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:10,707 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:10,707 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:10,707 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:10,707 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:10,708 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=727 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/A/dadc7fdfe9204f92897e903c250815e0 2024-12-04T19:14:10,715 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 97532d2585300ca10ffc4f6d5dda82c5/C of 97532d2585300ca10ffc4f6d5dda82c5 into 42e2fc4952e74da5b14fb810f5f46417(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:14:10,715 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 97532d2585300ca10ffc4f6d5dda82c5: 2024-12-04T19:14:10,715 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., storeName=97532d2585300ca10ffc4f6d5dda82c5/C, priority=13, startTime=1733339650183; duration=0sec 2024-12-04T19:14:10,716 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:14:10,716 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 97532d2585300ca10ffc4f6d5dda82c5:C 2024-12-04T19:14:10,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/B/749ffd0cc5b5401f9ce8b3cc42ba8374 is 50, key is test_row_0/B:col10/1733339649738/Put/seqid=0 2024-12-04T19:14:10,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741928_1104 (size=12301) 2024-12-04T19:14:11,003 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:11,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57662 deadline: 1733339711002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:11,004 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:11,004 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:11,004 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:11,014 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:11,014 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:11,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57646 deadline: 1733339711012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:11,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57632 deadline: 1733339711012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:11,015 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:11,015 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:11,015 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:11,015 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:11,015 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:11,015 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:11,015 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:11,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57588 deadline: 1733339711012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:11,015 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:11,015 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:11,015 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:11,127 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=727 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/B/749ffd0cc5b5401f9ce8b3cc42ba8374 2024-12-04T19:14:11,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/C/efbf054ccfa741d8888bcaf8e61f2aad is 50, key is test_row_0/C:col10/1733339649738/Put/seqid=0 2024-12-04T19:14:11,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741929_1105 (size=12301) 2024-12-04T19:14:11,141 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=727 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/C/efbf054ccfa741d8888bcaf8e61f2aad 2024-12-04T19:14:11,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/A/dadc7fdfe9204f92897e903c250815e0 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/dadc7fdfe9204f92897e903c250815e0 2024-12-04T19:14:11,152 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/dadc7fdfe9204f92897e903c250815e0, entries=150, sequenceid=727, filesize=12.0 K 2024-12-04T19:14:11,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/B/749ffd0cc5b5401f9ce8b3cc42ba8374 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/749ffd0cc5b5401f9ce8b3cc42ba8374 2024-12-04T19:14:11,162 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/749ffd0cc5b5401f9ce8b3cc42ba8374, entries=150, sequenceid=727, filesize=12.0 K 2024-12-04T19:14:11,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/C/efbf054ccfa741d8888bcaf8e61f2aad as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/efbf054ccfa741d8888bcaf8e61f2aad 2024-12-04T19:14:11,170 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/efbf054ccfa741d8888bcaf8e61f2aad, entries=150, sequenceid=727, filesize=12.0 K 2024-12-04T19:14:11,171 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~430.64 KB/440976, currentSize=33.54 KB/34350 for 97532d2585300ca10ffc4f6d5dda82c5 in 889ms, sequenceid=727, compaction requested=false 2024-12-04T19:14:11,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 97532d2585300ca10ffc4f6d5dda82c5: 2024-12-04T19:14:11,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:11,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-12-04T19:14:11,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-12-04T19:14:11,174 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-12-04T19:14:11,174 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1180 sec 2024-12-04T19:14:11,177 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 2.1230 sec 2024-12-04T19:14:11,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-04T19:14:11,202 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T19:14:11,202 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T19:14:11,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-12-04T19:14:11,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-04T19:14:11,207 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T19:14:11,209 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T19:14:11,209 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T19:14:11,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-04T19:14:11,362 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-04T19:14:11,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:11,362 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 97532d2585300ca10ffc4f6d5dda82c5 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-04T19:14:11,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97532d2585300ca10ffc4f6d5dda82c5, store=A 2024-12-04T19:14:11,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:14:11,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97532d2585300ca10ffc4f6d5dda82c5, store=B 2024-12-04T19:14:11,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:14:11,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97532d2585300ca10ffc4f6d5dda82c5, store=C 2024-12-04T19:14:11,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:14:11,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/A/f0cf0c8e174c46ef97aa3950baf03824 is 50, key is test_row_0/A:col10/1733339650373/Put/seqid=0 2024-12-04T19:14:11,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741930_1106 (size=9857) 2024-12-04T19:14:11,385 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=737 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/A/f0cf0c8e174c46ef97aa3950baf03824 2024-12-04T19:14:11,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/B/a49a3a4cf18143b09c35e06b4fede94b is 50, key is test_row_0/B:col10/1733339650373/Put/seqid=0 2024-12-04T19:14:11,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741931_1107 (size=9857) 2024-12-04T19:14:11,516 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. as already flushing 2024-12-04T19:14:11,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(8855): Flush requested on 97532d2585300ca10ffc4f6d5dda82c5 2024-12-04T19:14:11,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-04T19:14:11,544 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733339651526, max timestamp=1733339651540Num uniques 859; 2024-12-04T19:14:11,544 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733339651526, max timestamp=1733339651540Num uniques 859; 2024-12-04T19:14:11,544 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733339651526, max timestamp=1733339651540Num uniques 859; 2024-12-04T19:14:11,544 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T19:14:11,544 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T19:14:11,544 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T19:14:11,572 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733339651561, max timestamp=1733339651568Num uniques 1018; 2024-12-04T19:14:11,573 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-04T19:14:11,573 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733339651561, max timestamp=1733339651568Num uniques 1018; 2024-12-04T19:14:11,573 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733339651561, max timestamp=1733339651568Num uniques 1018; 2024-12-04T19:14:11,573 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-04T19:14:11,573 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-04T19:14:11,643 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733339651628, max timestamp=1733339651640Num uniques 1027; 2024-12-04T19:14:11,643 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-12-04T19:14:11,643 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733339651628, max timestamp=1733339651640Num uniques 1027; 2024-12-04T19:14:11,643 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-12-04T19:14:11,645 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733339651628, max timestamp=1733339651640Num uniques 1027; 2024-12-04T19:14:11,645 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-12-04T19:14:11,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:11,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57662 deadline: 1733339711665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:11,673 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:11,673 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:11,673 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:11,674 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1733339651654, max timestamp=1733339651670Num uniques 1086; 2024-12-04T19:14:11,674 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13086, totalHeapSize=29984, min timestamp=1733339651654, max timestamp=1733339651671Num uniques 1086; 2024-12-04T19:14:11,674 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=13096, suffix heap size=140912,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=286 2024-12-04T19:14:11,674 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49726, new segment data size=13086, suffix heap size=140912,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=286 2024-12-04T19:14:11,674 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1733339651654, max timestamp=1733339651670Num uniques 1086; 2024-12-04T19:14:11,674 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=13096, suffix heap size=140912,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=286 2024-12-04T19:14:11,706 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:11,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57588 deadline: 1733339711701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:11,707 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:11,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57632 deadline: 1733339711702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:11,707 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:11,707 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:11,707 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:11,707 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:11,707 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:11,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 224 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57646 deadline: 1733339711704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:11,707 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:11,707 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:11,708 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:11,708 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:11,708 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:11,783 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:11,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57662 deadline: 1733339711781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:11,784 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:11,784 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:11,784 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:11,807 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=737 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/B/a49a3a4cf18143b09c35e06b4fede94b 2024-12-04T19:14:11,814 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:11,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 217 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57632 deadline: 1733339711812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:11,815 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:11,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57588 deadline: 1733339711812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:11,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:11,815 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:11,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 225 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57646 deadline: 1733339711812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:11,815 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:11,815 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:11,815 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:11,815 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:11,815 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:11,815 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:11,815 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:11,815 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:11,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/C/47e80d2ae8694dc2ba4383513c6298fd is 50, key is test_row_0/C:col10/1733339650373/Put/seqid=0 2024-12-04T19:14:11,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741932_1108 (size=9857) 2024-12-04T19:14:11,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-04T19:14:11,994 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:11,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57662 deadline: 1733339711992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:11,995 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:11,995 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:11,995 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:12,023 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:12,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 218 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57632 deadline: 1733339712022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:12,024 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:12,024 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:12,024 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:12,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57588 deadline: 1733339712022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:12,024 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:12,024 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:12,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 226 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57646 deadline: 1733339712022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:12,024 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:12,024 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:12,024 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:12,025 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:12,025 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:12,025 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:12,231 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=737 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/C/47e80d2ae8694dc2ba4383513c6298fd 2024-12-04T19:14:12,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/A/f0cf0c8e174c46ef97aa3950baf03824 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/f0cf0c8e174c46ef97aa3950baf03824 2024-12-04T19:14:12,240 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/f0cf0c8e174c46ef97aa3950baf03824, entries=100, sequenceid=737, filesize=9.6 K 2024-12-04T19:14:12,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/B/a49a3a4cf18143b09c35e06b4fede94b as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/a49a3a4cf18143b09c35e06b4fede94b 2024-12-04T19:14:12,248 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/a49a3a4cf18143b09c35e06b4fede94b, entries=100, sequenceid=737, filesize=9.6 K 2024-12-04T19:14:12,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/C/47e80d2ae8694dc2ba4383513c6298fd as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/47e80d2ae8694dc2ba4383513c6298fd 2024-12-04T19:14:12,256 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/47e80d2ae8694dc2ba4383513c6298fd, entries=100, sequenceid=737, filesize=9.6 K 2024-12-04T19:14:12,258 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=147.60 KB/151140 for 97532d2585300ca10ffc4f6d5dda82c5 in 895ms, sequenceid=737, compaction requested=true 2024-12-04T19:14:12,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 97532d2585300ca10ffc4f6d5dda82c5: 2024-12-04T19:14:12,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:12,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-12-04T19:14:12,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-12-04T19:14:12,261 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-12-04T19:14:12,261 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0500 sec 2024-12-04T19:14:12,263 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.0600 sec 2024-12-04T19:14:12,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(8855): Flush requested on 97532d2585300ca10ffc4f6d5dda82c5 2024-12-04T19:14:12,304 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 97532d2585300ca10ffc4f6d5dda82c5 3/3 column families, dataSize=154.31 KB heapSize=433.55 KB 2024-12-04T19:14:12,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97532d2585300ca10ffc4f6d5dda82c5, store=A 2024-12-04T19:14:12,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-04T19:14:12,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97532d2585300ca10ffc4f6d5dda82c5, store=B 2024-12-04T19:14:12,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-04T19:14:12,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97532d2585300ca10ffc4f6d5dda82c5, store=C 2024-12-04T19:14:12,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-04T19:14:12,310 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/A/705b5848e50745529c51b041ef849e14 is 50, key is test_row_0/A:col10/1733339651670/Put/seqid=0 2024-12-04T19:14:12,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741933_1109 (size=14741) 2024-12-04T19:14:12,318 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=826 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/A/705b5848e50745529c51b041ef849e14 2024-12-04T19:14:12,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:12,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57662 deadline: 1733339712319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:12,322 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:12,322 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:12,322 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:12,329 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/B/66a0a47638dd41bf96de5b6a8d20d1ea is 50, key is test_row_0/B:col10/1733339651670/Put/seqid=0 2024-12-04T19:14:12,334 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:12,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 219 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57632 deadline: 1733339712332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:12,334 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:12,334 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:12,334 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:12,334 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:12,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 227 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57646 deadline: 1733339712332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:12,335 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:12,335 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:12,335 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:12,335 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:12,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57588 deadline: 1733339712332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:12,335 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:12,335 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:12,335 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:12,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741934_1110 (size=12301) 2024-12-04T19:14:12,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-04T19:14:12,342 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T19:14:12,342 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T19:14:12,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-12-04T19:14:12,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-04T19:14:12,344 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T19:14:12,345 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T19:14:12,345 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T19:14:12,433 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:12,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57662 deadline: 1733339712432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:12,434 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:12,434 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:12,434 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:12,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-04T19:14:12,497 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-04T19:14:12,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:12,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. as already flushing 2024-12-04T19:14:12,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:12,497 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:12,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:12,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:12,644 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:12,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57662 deadline: 1733339712642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:12,645 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:12,645 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:12,645 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:12,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-04T19:14:12,664 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-04T19:14:12,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:12,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. as already flushing 2024-12-04T19:14:12,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:12,664 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:12,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:12,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:12,739 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=826 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/B/66a0a47638dd41bf96de5b6a8d20d1ea 2024-12-04T19:14:12,761 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/C/c1cdd9270db24764bf0a2988f8f4d8cc is 50, key is test_row_0/C:col10/1733339651670/Put/seqid=0 2024-12-04T19:14:12,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741935_1111 (size=12301) 2024-12-04T19:14:12,772 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=826 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/C/c1cdd9270db24764bf0a2988f8f4d8cc 2024-12-04T19:14:12,779 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/A/705b5848e50745529c51b041ef849e14 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/705b5848e50745529c51b041ef849e14 2024-12-04T19:14:12,784 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/705b5848e50745529c51b041ef849e14, entries=200, sequenceid=826, filesize=14.4 K 2024-12-04T19:14:12,785 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/B/66a0a47638dd41bf96de5b6a8d20d1ea as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/66a0a47638dd41bf96de5b6a8d20d1ea 2024-12-04T19:14:12,790 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/66a0a47638dd41bf96de5b6a8d20d1ea, entries=150, sequenceid=826, filesize=12.0 K 2024-12-04T19:14:12,791 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/C/c1cdd9270db24764bf0a2988f8f4d8cc as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/c1cdd9270db24764bf0a2988f8f4d8cc 2024-12-04T19:14:12,796 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/c1cdd9270db24764bf0a2988f8f4d8cc, entries=150, sequenceid=826, filesize=12.0 K 2024-12-04T19:14:12,797 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~433.50 KB/443904, currentSize=26.84 KB/27480 for 97532d2585300ca10ffc4f6d5dda82c5 in 494ms, sequenceid=826, compaction requested=true 2024-12-04T19:14:12,797 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 97532d2585300ca10ffc4f6d5dda82c5: 2024-12-04T19:14:12,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 97532d2585300ca10ffc4f6d5dda82c5:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T19:14:12,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:14:12,797 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T19:14:12,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 97532d2585300ca10ffc4f6d5dda82c5:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T19:14:12,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:14:12,797 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T19:14:12,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 97532d2585300ca10ffc4f6d5dda82c5:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T19:14:12,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T19:14:12,799 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47748 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T19:14:12,799 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50188 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T19:14:12,799 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1541): 97532d2585300ca10ffc4f6d5dda82c5/A is initiating minor compaction (all files) 2024-12-04T19:14:12,799 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1541): 97532d2585300ca10ffc4f6d5dda82c5/B is initiating minor compaction (all files) 2024-12-04T19:14:12,799 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 97532d2585300ca10ffc4f6d5dda82c5/A in TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:12,799 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 97532d2585300ca10ffc4f6d5dda82c5/B in TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:12,799 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/7269dc6260804590b7fedc61b386ae9e, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/749ffd0cc5b5401f9ce8b3cc42ba8374, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/a49a3a4cf18143b09c35e06b4fede94b, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/66a0a47638dd41bf96de5b6a8d20d1ea] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp, totalSize=46.6 K 2024-12-04T19:14:12,799 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/ca95ba3aa611478cb4b5b6d185dbc125, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/dadc7fdfe9204f92897e903c250815e0, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/f0cf0c8e174c46ef97aa3950baf03824, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/705b5848e50745529c51b041ef849e14] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp, totalSize=49.0 K 2024-12-04T19:14:12,799 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting ca95ba3aa611478cb4b5b6d185dbc125, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=689, earliestPutTs=1733339649532 2024-12-04T19:14:12,799 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 7269dc6260804590b7fedc61b386ae9e, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=689, earliestPutTs=1733339649532 2024-12-04T19:14:12,800 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting dadc7fdfe9204f92897e903c250815e0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=727, earliestPutTs=1733339649734 2024-12-04T19:14:12,800 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 749ffd0cc5b5401f9ce8b3cc42ba8374, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=727, earliestPutTs=1733339649734 2024-12-04T19:14:12,800 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting f0cf0c8e174c46ef97aa3950baf03824, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=737, earliestPutTs=1733339650373 2024-12-04T19:14:12,800 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting a49a3a4cf18143b09c35e06b4fede94b, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=737, earliestPutTs=1733339650373 2024-12-04T19:14:12,800 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 705b5848e50745529c51b041ef849e14, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=826, earliestPutTs=1733339651670 2024-12-04T19:14:12,800 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 66a0a47638dd41bf96de5b6a8d20d1ea, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=826, earliestPutTs=1733339651670 2024-12-04T19:14:12,810 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 97532d2585300ca10ffc4f6d5dda82c5#A#compaction#99 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:14:12,811 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/A/42f8b677132d45488fb6def42902618a is 50, key is test_row_0/A:col10/1733339651670/Put/seqid=0 2024-12-04T19:14:12,816 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 97532d2585300ca10ffc4f6d5dda82c5#B#compaction#100 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:14:12,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741936_1112 (size=13425) 2024-12-04T19:14:12,817 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/B/c0cff809fe5042d290ca31c505398583 is 50, key is test_row_0/B:col10/1733339651670/Put/seqid=0 2024-12-04T19:14:12,817 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-04T19:14:12,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:12,817 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 97532d2585300ca10ffc4f6d5dda82c5 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-12-04T19:14:12,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97532d2585300ca10ffc4f6d5dda82c5, store=A 2024-12-04T19:14:12,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:14:12,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97532d2585300ca10ffc4f6d5dda82c5, store=B 2024-12-04T19:14:12,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:14:12,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97532d2585300ca10ffc4f6d5dda82c5, store=C 2024-12-04T19:14:12,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:14:12,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/A/9f29bc453a894d05a74de59d3c5ad8a3 is 50, key is test_row_0/A:col10/1733339652317/Put/seqid=0 2024-12-04T19:14:12,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741937_1113 (size=13425) 2024-12-04T19:14:12,830 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/A/42f8b677132d45488fb6def42902618a as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/42f8b677132d45488fb6def42902618a 2024-12-04T19:14:12,838 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 97532d2585300ca10ffc4f6d5dda82c5/A of 97532d2585300ca10ffc4f6d5dda82c5 into 42f8b677132d45488fb6def42902618a(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:14:12,838 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 97532d2585300ca10ffc4f6d5dda82c5: 2024-12-04T19:14:12,838 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., storeName=97532d2585300ca10ffc4f6d5dda82c5/A, priority=12, startTime=1733339652797; duration=0sec 2024-12-04T19:14:12,838 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T19:14:12,838 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 97532d2585300ca10ffc4f6d5dda82c5:A 2024-12-04T19:14:12,838 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T19:14:12,847 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47748 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T19:14:12,847 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/B/c0cff809fe5042d290ca31c505398583 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/c0cff809fe5042d290ca31c505398583 2024-12-04T19:14:12,847 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1541): 97532d2585300ca10ffc4f6d5dda82c5/C is initiating minor compaction (all files) 2024-12-04T19:14:12,847 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 97532d2585300ca10ffc4f6d5dda82c5/C in TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:12,847 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/42e2fc4952e74da5b14fb810f5f46417, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/efbf054ccfa741d8888bcaf8e61f2aad, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/47e80d2ae8694dc2ba4383513c6298fd, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/c1cdd9270db24764bf0a2988f8f4d8cc] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp, totalSize=46.6 K 2024-12-04T19:14:12,849 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 42e2fc4952e74da5b14fb810f5f46417, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=689, earliestPutTs=1733339649532 2024-12-04T19:14:12,850 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting efbf054ccfa741d8888bcaf8e61f2aad, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=727, earliestPutTs=1733339649734 2024-12-04T19:14:12,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741938_1114 (size=12301) 2024-12-04T19:14:12,850 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 47e80d2ae8694dc2ba4383513c6298fd, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=737, earliestPutTs=1733339650373 2024-12-04T19:14:12,850 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting c1cdd9270db24764bf0a2988f8f4d8cc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=826, earliestPutTs=1733339651670 2024-12-04T19:14:12,862 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 97532d2585300ca10ffc4f6d5dda82c5#C#compaction#102 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:14:12,862 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/C/9034b21a97844625928ee0df60d8cf53 is 50, key is test_row_0/C:col10/1733339651670/Put/seqid=0 2024-12-04T19:14:12,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741939_1115 (size=13425) 2024-12-04T19:14:12,874 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 97532d2585300ca10ffc4f6d5dda82c5/B of 97532d2585300ca10ffc4f6d5dda82c5 into c0cff809fe5042d290ca31c505398583(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:14:12,874 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 97532d2585300ca10ffc4f6d5dda82c5: 2024-12-04T19:14:12,874 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., storeName=97532d2585300ca10ffc4f6d5dda82c5/B, priority=12, startTime=1733339652797; duration=0sec 2024-12-04T19:14:12,874 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:14:12,874 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 97532d2585300ca10ffc4f6d5dda82c5:B 2024-12-04T19:14:12,874 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. as already flushing 2024-12-04T19:14:12,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(8855): Flush requested on 97532d2585300ca10ffc4f6d5dda82c5 2024-12-04T19:14:12,909 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733339652890, max timestamp=1733339652907Num uniques 859; 2024-12-04T19:14:12,909 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733339652890, max timestamp=1733339652907Num uniques 859; 2024-12-04T19:14:12,909 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T19:14:12,909 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T19:14:12,910 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733339652890, max timestamp=1733339652907Num uniques 859; 2024-12-04T19:14:12,910 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T19:14:12,952 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1733339652929, max timestamp=1733339652949Num uniques 1068; 2024-12-04T19:14:12,952 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=12268, suffix heap size=139040,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=268 2024-12-04T19:14:12,953 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1733339652929, max timestamp=1733339652949Num uniques 1068; 2024-12-04T19:14:12,953 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1733339652929, max timestamp=1733339652949Num uniques 1068; 2024-12-04T19:14:12,953 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=12268, suffix heap size=139040,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=268 2024-12-04T19:14:12,953 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=12268, suffix heap size=139040,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=268 2024-12-04T19:14:12,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-04T19:14:13,005 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=327, cellSize=14972, totalHeapSize=34248, min timestamp=1733339652970, max timestamp=1733339653001Num uniques 1127; 2024-12-04T19:14:13,005 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=327, cellSize=14972, totalHeapSize=34248, min timestamp=1733339652970, max timestamp=1733339653001Num uniques 1127; 2024-12-04T19:14:13,005 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=14972, suffix heap size=145176,new segment heap size=34248 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=327 2024-12-04T19:14:13,005 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=14972, suffix heap size=145176,new segment heap size=34248 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=327 2024-12-04T19:14:13,006 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=327, cellSize=14972, totalHeapSize=34248, min timestamp=1733339652970, max timestamp=1733339653001Num uniques 1127; 2024-12-04T19:14:13,006 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=14972, suffix heap size=145176,new segment heap size=34248 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=327 2024-12-04T19:14:13,033 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:13,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 246 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57646 deadline: 1733339713024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:13,034 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:13,034 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:13,034 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:13,035 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=336, cellSize=15386, totalHeapSize=35184, min timestamp=1733339653016, max timestamp=1733339653029Num uniques 1186; 2024-12-04T19:14:13,035 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=54316, new segment data size=15386, suffix heap size=151312,new segment heap size=35184 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1186, new segment cells count=336 2024-12-04T19:14:13,035 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=336, cellSize=15386, totalHeapSize=35184, min timestamp=1733339653016, max timestamp=1733339653029Num uniques 1186; 2024-12-04T19:14:13,035 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=54316, new segment data size=15386, suffix heap size=151312,new segment heap size=35184 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1186, new segment cells count=336 2024-12-04T19:14:13,036 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=336, cellSize=15386, totalHeapSize=35184, min timestamp=1733339653016, max timestamp=1733339653029Num uniques 1186; 2024-12-04T19:14:13,036 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=54316, new segment data size=15386, suffix heap size=151312,new segment heap size=35184 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1186, new segment cells count=336 2024-12-04T19:14:13,075 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:13,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 231 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57662 deadline: 1733339713071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:13,075 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:13,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 248 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57632 deadline: 1733339713072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:13,076 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:13,076 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:13,076 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:13,077 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1733339653060, max timestamp=1733339653072Num uniques 1195; 2024-12-04T19:14:13,077 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=54730, new segment data size=8930, suffix heap size=152248,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1195, new segment cells count=195 2024-12-04T19:14:13,079 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1733339653060, max timestamp=1733339653072Num uniques 1195; 2024-12-04T19:14:13,079 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=54730, new segment data size=8930, suffix heap size=152248,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1195, new segment cells count=195 2024-12-04T19:14:13,082 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1733339653060, max timestamp=1733339653072Num uniques 1195; 2024-12-04T19:14:13,082 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=54730, new segment data size=8930, suffix heap size=152248,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1195, new segment cells count=195 2024-12-04T19:14:13,082 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:13,082 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:13,082 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:13,152 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1733339653093, max timestamp=1733339653148Num uniques 1054; 2024-12-04T19:14:13,152 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1733339653093, max timestamp=1733339653149Num uniques 1054; 2024-12-04T19:14:13,152 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1733339653093, max timestamp=1733339653149Num uniques 1054; 2024-12-04T19:14:13,152 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=9340, suffix heap size=137584,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=204 2024-12-04T19:14:13,152 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=9340, suffix heap size=137584,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=204 2024-12-04T19:14:13,152 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=9340, suffix heap size=137584,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=204 2024-12-04T19:14:13,181 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1733339653169, max timestamp=1733339653179Num uniques 1063; 2024-12-04T19:14:13,181 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=9748, suffix heap size=138520,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=213 2024-12-04T19:14:13,182 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1733339653169, max timestamp=1733339653179Num uniques 1063; 2024-12-04T19:14:13,182 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=9748, suffix heap size=138520,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=213 2024-12-04T19:14:13,182 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1733339653169, max timestamp=1733339653179Num uniques 1063; 2024-12-04T19:14:13,182 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=9748, suffix heap size=138520,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=213 2024-12-04T19:14:13,207 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=322, cellSize=14742, totalHeapSize=33728, min timestamp=1733339653197, max timestamp=1733339653204Num uniques 1072; 2024-12-04T19:14:13,207 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=14742, suffix heap size=139456,new segment heap size=33728 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=322 2024-12-04T19:14:13,207 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=322, cellSize=14742, totalHeapSize=33728, min timestamp=1733339653197, max timestamp=1733339653204Num uniques 1072; 2024-12-04T19:14:13,207 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=322, cellSize=14742, totalHeapSize=33728, min timestamp=1733339653197, max timestamp=1733339653204Num uniques 1072; 2024-12-04T19:14:13,207 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=14742, suffix heap size=139456,new segment heap size=33728 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=322 2024-12-04T19:14:13,207 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=14742, suffix heap size=139456,new segment heap size=33728 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=322 2024-12-04T19:14:13,229 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:13,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 267 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57588 deadline: 1733339713226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:13,231 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:13,231 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=381, cellSize=17446, totalHeapSize=39864, min timestamp=1733339653219, max timestamp=1733339653228Num uniques 1181; 2024-12-04T19:14:13,231 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=381, cellSize=17446, totalHeapSize=39864, min timestamp=1733339653219, max timestamp=1733339653228Num uniques 1181; 2024-12-04T19:14:13,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 238 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57662 deadline: 1733339713227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:13,231 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=54086, new segment data size=17446, suffix heap size=150792,new segment heap size=39864 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1181, new segment cells count=381 2024-12-04T19:14:13,231 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=54086, new segment data size=17446, suffix heap size=150792,new segment heap size=39864 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1181, new segment cells count=381 2024-12-04T19:14:13,231 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=381, cellSize=17446, totalHeapSize=39864, min timestamp=1733339653219, max timestamp=1733339653228Num uniques 1181; 2024-12-04T19:14:13,232 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=54086, new segment data size=17446, suffix heap size=150792,new segment heap size=39864 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1181, new segment cells count=381 2024-12-04T19:14:13,232 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:13,232 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:13,232 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:13,232 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:13,232 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:13,232 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:13,251 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=833 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/A/9f29bc453a894d05a74de59d3c5ad8a3 2024-12-04T19:14:13,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/B/3bc71a1eb83d403abf2de17740dd6625 is 50, key is test_row_0/B:col10/1733339652317/Put/seqid=0 2024-12-04T19:14:13,269 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:13,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 266 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57632 deadline: 1733339713265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:13,269 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:13,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:13,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 275 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57646 deadline: 1733339713267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:13,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:13,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:13,270 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:13,270 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:13,270 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:13,277 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/C/9034b21a97844625928ee0df60d8cf53 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/9034b21a97844625928ee0df60d8cf53 2024-12-04T19:14:13,283 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 97532d2585300ca10ffc4f6d5dda82c5/C of 97532d2585300ca10ffc4f6d5dda82c5 into 9034b21a97844625928ee0df60d8cf53(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:14:13,283 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 97532d2585300ca10ffc4f6d5dda82c5: 2024-12-04T19:14:13,283 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., storeName=97532d2585300ca10ffc4f6d5dda82c5/C, priority=12, startTime=1733339652797; duration=0sec 2024-12-04T19:14:13,284 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:14:13,284 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 97532d2585300ca10ffc4f6d5dda82c5:C 2024-12-04T19:14:13,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741940_1116 (size=12301) 2024-12-04T19:14:13,343 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:13,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 239 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57662 deadline: 1733339713342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:13,344 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:13,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 268 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57588 deadline: 1733339713342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:13,344 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:13,344 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:13,344 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:13,344 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:13,344 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:13,344 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:13,373 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:13,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 276 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57646 deadline: 1733339713371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:13,374 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:13,374 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:13,374 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:13,382 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:13,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 267 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57632 deadline: 1733339713381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:13,383 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:13,383 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:13,383 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:13,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-04T19:14:13,555 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:13,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:13,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 240 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57662 deadline: 1733339713552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:13,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 269 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57588 deadline: 1733339713552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:13,556 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:13,556 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:13,556 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:13,556 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:13,556 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:13,556 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:13,582 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:13,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 277 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57646 deadline: 1733339713581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:13,583 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:13,583 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:13,583 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:13,593 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:13,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 268 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57632 deadline: 1733339713592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:13,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:13,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:13,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:13,696 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=833 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/B/3bc71a1eb83d403abf2de17740dd6625 2024-12-04T19:14:13,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/C/64089ceed5234119a0743305f92393f0 is 50, key is test_row_0/C:col10/1733339652317/Put/seqid=0 2024-12-04T19:14:13,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741941_1117 (size=12301) 2024-12-04T19:14:13,711 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=833 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/C/64089ceed5234119a0743305f92393f0 2024-12-04T19:14:13,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/A/9f29bc453a894d05a74de59d3c5ad8a3 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/9f29bc453a894d05a74de59d3c5ad8a3 2024-12-04T19:14:13,721 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/9f29bc453a894d05a74de59d3c5ad8a3, entries=150, sequenceid=833, filesize=12.0 K 2024-12-04T19:14:13,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/B/3bc71a1eb83d403abf2de17740dd6625 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/3bc71a1eb83d403abf2de17740dd6625 2024-12-04T19:14:13,728 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/3bc71a1eb83d403abf2de17740dd6625, entries=150, sequenceid=833, filesize=12.0 K 2024-12-04T19:14:13,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/C/64089ceed5234119a0743305f92393f0 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/64089ceed5234119a0743305f92393f0 2024-12-04T19:14:13,734 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/64089ceed5234119a0743305f92393f0, entries=150, sequenceid=833, filesize=12.0 K 2024-12-04T19:14:13,735 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=161.02 KB/164880 for 97532d2585300ca10ffc4f6d5dda82c5 in 918ms, sequenceid=833, compaction requested=false 2024-12-04T19:14:13,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 97532d2585300ca10ffc4f6d5dda82c5: 2024-12-04T19:14:13,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:13,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-12-04T19:14:13,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-12-04T19:14:13,738 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-12-04T19:14:13,738 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3910 sec 2024-12-04T19:14:13,740 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 1.3970 sec 2024-12-04T19:14:13,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(8855): Flush requested on 97532d2585300ca10ffc4f6d5dda82c5 2024-12-04T19:14:13,864 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 97532d2585300ca10ffc4f6d5dda82c5 3/3 column families, dataSize=174.43 KB heapSize=484.41 KB 2024-12-04T19:14:13,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97532d2585300ca10ffc4f6d5dda82c5, store=A 2024-12-04T19:14:13,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-04T19:14:13,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97532d2585300ca10ffc4f6d5dda82c5, store=B 2024-12-04T19:14:13,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-04T19:14:13,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97532d2585300ca10ffc4f6d5dda82c5, store=C 2024-12-04T19:14:13,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-04T19:14:13,870 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/A/dda2e7a3beef4071a8cfa38654269204 is 50, key is test_row_0/A:col10/1733339653862/Put/seqid=0 2024-12-04T19:14:13,872 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:13,872 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:13,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 272 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57588 deadline: 1733339713868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:13,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 243 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57662 deadline: 1733339713869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:13,873 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:13,873 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:13,873 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:13,876 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:13,876 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:13,876 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:13,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741942_1118 (size=12301) 2024-12-04T19:14:13,894 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:13,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 278 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57646 deadline: 1733339713891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:13,894 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:13,894 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:13,895 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:13,904 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:13,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 269 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57632 deadline: 1733339713901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:13,905 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:13,905 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:13,905 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:13,983 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:13,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 244 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57662 deadline: 1733339713981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:13,983 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:13,983 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:13,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 273 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57588 deadline: 1733339713981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:13,983 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:13,983 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:13,984 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:13,984 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:13,984 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:14,194 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:14,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 245 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57662 deadline: 1733339714192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:14,194 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:14,194 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:14,195 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:14,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 274 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57588 deadline: 1733339714192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:14,195 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:14,195 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:14,195 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:14,195 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:14,281 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=1012 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/A/dda2e7a3beef4071a8cfa38654269204 2024-12-04T19:14:14,289 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/B/49fc640ac0e54c3d9a996962a26ad92c is 50, key is test_row_0/B:col10/1733339653862/Put/seqid=0 2024-12-04T19:14:14,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741943_1119 (size=12301) 2024-12-04T19:14:14,402 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:14,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 279 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57646 deadline: 1733339714401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:14,402 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:14,402 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:14,402 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:14,413 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:14,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 270 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57632 deadline: 1733339714412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:14,413 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:14,413 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:14,413 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:14,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-04T19:14:14,492 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T19:14:14,492 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T19:14:14,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-12-04T19:14:14,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-04T19:14:14,494 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T19:14:14,494 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T19:14:14,494 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T19:14:14,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:14,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 246 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57662 deadline: 1733339714502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:14,503 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:14,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 275 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57588 deadline: 1733339714502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:14,504 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:14,504 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:14,504 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:14,504 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:14,504 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:14,504 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:14,593 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:14,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57600 deadline: 1733339714591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:14,593 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 9ba2734cd39f,33969,1733339629791 for 'test_row_1' in 97532d2585300ca10ffc4f6d5dda82c5 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8241 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-04T19:14:14,594 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:14,594 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:14,594 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:14,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-04T19:14:14,646 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-04T19:14:14,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:14,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. as already flushing 2024-12-04T19:14:14,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:14,647 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:14,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:14,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:14,693 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=1012 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/B/49fc640ac0e54c3d9a996962a26ad92c 2024-12-04T19:14:14,701 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/C/6aba5cadef98445c8a233f9ccc4a89fd is 50, key is test_row_0/C:col10/1733339653862/Put/seqid=0 2024-12-04T19:14:14,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741944_1120 (size=12301) 2024-12-04T19:14:14,798 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-04T19:14:14,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:14,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. as already flushing 2024-12-04T19:14:14,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:14,799 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:14,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:14,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:14,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-04T19:14:14,951 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-04T19:14:14,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:14,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. as already flushing 2024-12-04T19:14:14,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:14,952 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:14,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:14,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:15,012 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:15,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 247 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57662 deadline: 1733339715012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:15,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:15,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 276 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57588 deadline: 1733339715012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:15,013 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:15,013 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:15,013 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:15,013 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:15,013 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:15,013 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:15,104 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-04T19:14:15,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:15,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. as already flushing 2024-12-04T19:14:15,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:15,104 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:15,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:15,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:15,106 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=1012 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/C/6aba5cadef98445c8a233f9ccc4a89fd 2024-12-04T19:14:15,111 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/A/dda2e7a3beef4071a8cfa38654269204 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/dda2e7a3beef4071a8cfa38654269204 2024-12-04T19:14:15,115 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/dda2e7a3beef4071a8cfa38654269204, entries=150, sequenceid=1012, filesize=12.0 K 2024-12-04T19:14:15,116 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/B/49fc640ac0e54c3d9a996962a26ad92c as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/49fc640ac0e54c3d9a996962a26ad92c 2024-12-04T19:14:15,121 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/49fc640ac0e54c3d9a996962a26ad92c, entries=150, sequenceid=1012, filesize=12.0 K 2024-12-04T19:14:15,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-04T19:14:15,124 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/C/6aba5cadef98445c8a233f9ccc4a89fd as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/6aba5cadef98445c8a233f9ccc4a89fd 2024-12-04T19:14:15,130 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/6aba5cadef98445c8a233f9ccc4a89fd, entries=150, sequenceid=1012, filesize=12.0 K 2024-12-04T19:14:15,131 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~174.43 KB/178620, heapSize ~484.36 KB/495984, currentSize=13.42 KB/13740 for 97532d2585300ca10ffc4f6d5dda82c5 in 1267ms, sequenceid=1012, compaction requested=true 2024-12-04T19:14:15,131 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 97532d2585300ca10ffc4f6d5dda82c5: 2024-12-04T19:14:15,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 97532d2585300ca10ffc4f6d5dda82c5:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T19:14:15,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:14:15,131 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T19:14:15,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 97532d2585300ca10ffc4f6d5dda82c5:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T19:14:15,131 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T19:14:15,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:14:15,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 97532d2585300ca10ffc4f6d5dda82c5:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T19:14:15,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T19:14:15,134 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38027 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T19:14:15,134 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1541): 97532d2585300ca10ffc4f6d5dda82c5/B is initiating minor compaction (all files) 2024-12-04T19:14:15,134 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 97532d2585300ca10ffc4f6d5dda82c5/B in TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:15,135 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/c0cff809fe5042d290ca31c505398583, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/3bc71a1eb83d403abf2de17740dd6625, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/49fc640ac0e54c3d9a996962a26ad92c] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp, totalSize=37.1 K 2024-12-04T19:14:15,137 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting c0cff809fe5042d290ca31c505398583, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=826, earliestPutTs=1733339651670 2024-12-04T19:14:15,138 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 3bc71a1eb83d403abf2de17740dd6625, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=833, earliestPutTs=1733339652307 2024-12-04T19:14:15,138 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38027 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T19:14:15,138 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1541): 97532d2585300ca10ffc4f6d5dda82c5/A is initiating minor compaction (all files) 2024-12-04T19:14:15,138 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 97532d2585300ca10ffc4f6d5dda82c5/A in TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:15,138 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/42f8b677132d45488fb6def42902618a, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/9f29bc453a894d05a74de59d3c5ad8a3, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/dda2e7a3beef4071a8cfa38654269204] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp, totalSize=37.1 K 2024-12-04T19:14:15,138 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 49fc640ac0e54c3d9a996962a26ad92c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=1012, earliestPutTs=1733339653265 2024-12-04T19:14:15,140 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 42f8b677132d45488fb6def42902618a, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=826, earliestPutTs=1733339651670 2024-12-04T19:14:15,140 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9f29bc453a894d05a74de59d3c5ad8a3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=833, earliestPutTs=1733339652307 2024-12-04T19:14:15,141 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting dda2e7a3beef4071a8cfa38654269204, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=1012, earliestPutTs=1733339653265 2024-12-04T19:14:15,150 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 97532d2585300ca10ffc4f6d5dda82c5#B#compaction#108 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:14:15,151 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/B/399abbcee1f346b5994fcff6c8baf7dc is 50, key is test_row_0/B:col10/1733339653862/Put/seqid=0 2024-12-04T19:14:15,154 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 97532d2585300ca10ffc4f6d5dda82c5#A#compaction#109 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:14:15,155 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/A/a0d456cc73b74372a8db3e71edd34ec8 is 50, key is test_row_0/A:col10/1733339653862/Put/seqid=0 2024-12-04T19:14:15,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741945_1121 (size=13527) 2024-12-04T19:14:15,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741946_1122 (size=13527) 2024-12-04T19:14:15,179 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/B/399abbcee1f346b5994fcff6c8baf7dc as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/399abbcee1f346b5994fcff6c8baf7dc 2024-12-04T19:14:15,184 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 97532d2585300ca10ffc4f6d5dda82c5/B of 97532d2585300ca10ffc4f6d5dda82c5 into 399abbcee1f346b5994fcff6c8baf7dc(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:14:15,184 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 97532d2585300ca10ffc4f6d5dda82c5: 2024-12-04T19:14:15,184 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., storeName=97532d2585300ca10ffc4f6d5dda82c5/B, priority=13, startTime=1733339655131; duration=0sec 2024-12-04T19:14:15,185 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T19:14:15,185 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 97532d2585300ca10ffc4f6d5dda82c5:B 2024-12-04T19:14:15,185 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T19:14:15,188 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38027 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T19:14:15,188 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1541): 97532d2585300ca10ffc4f6d5dda82c5/C is initiating minor compaction (all files) 2024-12-04T19:14:15,188 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 97532d2585300ca10ffc4f6d5dda82c5/C in TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:15,188 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/9034b21a97844625928ee0df60d8cf53, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/64089ceed5234119a0743305f92393f0, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/6aba5cadef98445c8a233f9ccc4a89fd] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp, totalSize=37.1 K 2024-12-04T19:14:15,189 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 9034b21a97844625928ee0df60d8cf53, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=826, earliestPutTs=1733339651670 2024-12-04T19:14:15,189 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 64089ceed5234119a0743305f92393f0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=833, earliestPutTs=1733339652307 2024-12-04T19:14:15,190 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 6aba5cadef98445c8a233f9ccc4a89fd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=1012, earliestPutTs=1733339653265 2024-12-04T19:14:15,199 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 97532d2585300ca10ffc4f6d5dda82c5#C#compaction#110 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:14:15,200 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/C/b48c55df66754366bb8ce7687eae8557 is 50, key is test_row_0/C:col10/1733339653862/Put/seqid=0 2024-12-04T19:14:15,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741947_1123 (size=13527) 2024-12-04T19:14:15,216 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/C/b48c55df66754366bb8ce7687eae8557 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/b48c55df66754366bb8ce7687eae8557 2024-12-04T19:14:15,223 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 97532d2585300ca10ffc4f6d5dda82c5/C of 97532d2585300ca10ffc4f6d5dda82c5 into b48c55df66754366bb8ce7687eae8557(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:14:15,224 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 97532d2585300ca10ffc4f6d5dda82c5: 2024-12-04T19:14:15,224 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., storeName=97532d2585300ca10ffc4f6d5dda82c5/C, priority=13, startTime=1733339655131; duration=0sec 2024-12-04T19:14:15,224 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:14:15,224 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 97532d2585300ca10ffc4f6d5dda82c5:C 2024-12-04T19:14:15,257 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-04T19:14:15,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:15,257 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing 97532d2585300ca10ffc4f6d5dda82c5 3/3 column families, dataSize=13.42 KB heapSize=40.59 KB 2024-12-04T19:14:15,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97532d2585300ca10ffc4f6d5dda82c5, store=A 2024-12-04T19:14:15,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:14:15,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97532d2585300ca10ffc4f6d5dda82c5, store=B 2024-12-04T19:14:15,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:14:15,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97532d2585300ca10ffc4f6d5dda82c5, store=C 2024-12-04T19:14:15,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:14:15,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/A/0cec263b854d4c918259efbb6a008e04 is 50, key is test_row_0/A:col10/1733339653868/Put/seqid=0 2024-12-04T19:14:15,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741948_1124 (size=9857) 2024-12-04T19:14:15,270 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=1019 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/A/0cec263b854d4c918259efbb6a008e04 2024-12-04T19:14:15,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/B/73d1166548d94b7e9941bb2f0efdffcc is 50, key is test_row_0/B:col10/1733339653868/Put/seqid=0 2024-12-04T19:14:15,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741949_1125 (size=9857) 2024-12-04T19:14:15,287 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=1019 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/B/73d1166548d94b7e9941bb2f0efdffcc 2024-12-04T19:14:15,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/C/22a1d9d85b344cccbdff5bc47f47a415 is 50, key is test_row_0/C:col10/1733339653868/Put/seqid=0 2024-12-04T19:14:15,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741950_1126 (size=9857) 2024-12-04T19:14:15,308 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=1019 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/C/22a1d9d85b344cccbdff5bc47f47a415 2024-12-04T19:14:15,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/A/0cec263b854d4c918259efbb6a008e04 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/0cec263b854d4c918259efbb6a008e04 2024-12-04T19:14:15,322 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/0cec263b854d4c918259efbb6a008e04, entries=100, sequenceid=1019, filesize=9.6 K 2024-12-04T19:14:15,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/B/73d1166548d94b7e9941bb2f0efdffcc as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/73d1166548d94b7e9941bb2f0efdffcc 2024-12-04T19:14:15,329 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/73d1166548d94b7e9941bb2f0efdffcc, entries=100, sequenceid=1019, filesize=9.6 K 2024-12-04T19:14:15,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/C/22a1d9d85b344cccbdff5bc47f47a415 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/22a1d9d85b344cccbdff5bc47f47a415 2024-12-04T19:14:15,337 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/22a1d9d85b344cccbdff5bc47f47a415, entries=100, sequenceid=1019, filesize=9.6 K 2024-12-04T19:14:15,339 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for 97532d2585300ca10ffc4f6d5dda82c5 in 81ms, sequenceid=1019, compaction requested=false 2024-12-04T19:14:15,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for 97532d2585300ca10ffc4f6d5dda82c5: 2024-12-04T19:14:15,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:15,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-12-04T19:14:15,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-12-04T19:14:15,343 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-12-04T19:14:15,343 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 847 msec 2024-12-04T19:14:15,345 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 851 msec 2024-12-04T19:14:15,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(8855): Flush requested on 97532d2585300ca10ffc4f6d5dda82c5 2024-12-04T19:14:15,420 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 97532d2585300ca10ffc4f6d5dda82c5 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-04T19:14:15,421 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97532d2585300ca10ffc4f6d5dda82c5, store=A 2024-12-04T19:14:15,421 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:14:15,421 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97532d2585300ca10ffc4f6d5dda82c5, store=B 2024-12-04T19:14:15,421 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:14:15,421 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97532d2585300ca10ffc4f6d5dda82c5, store=C 2024-12-04T19:14:15,421 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:14:15,426 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/A/76b3d4c1d4c14371a80b2872183bd32a is 50, key is test_row_0/A:col10/1733339655416/Put/seqid=0 2024-12-04T19:14:15,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741951_1127 (size=14741) 2024-12-04T19:14:15,456 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733339655438, max timestamp=1733339655454Num uniques 859; 2024-12-04T19:14:15,456 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-04T19:14:15,457 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733339655438, max timestamp=1733339655454Num uniques 859; 2024-12-04T19:14:15,457 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-04T19:14:15,457 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733339655438, max timestamp=1733339655454Num uniques 859; 2024-12-04T19:14:15,457 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-04T19:14:15,488 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:15,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 303 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57646 deadline: 1733339715484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:15,489 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:15,489 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:15,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 287 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57632 deadline: 1733339715485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:15,489 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:15,489 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:15,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:15,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:15,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:15,578 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/A/a0d456cc73b74372a8db3e71edd34ec8 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/a0d456cc73b74372a8db3e71edd34ec8 2024-12-04T19:14:15,584 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 97532d2585300ca10ffc4f6d5dda82c5/A of 97532d2585300ca10ffc4f6d5dda82c5 into a0d456cc73b74372a8db3e71edd34ec8(size=13.2 K), total size for store is 22.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:14:15,584 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 97532d2585300ca10ffc4f6d5dda82c5: 2024-12-04T19:14:15,584 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., storeName=97532d2585300ca10ffc4f6d5dda82c5/A, priority=13, startTime=1733339655131; duration=0sec 2024-12-04T19:14:15,584 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:14:15,584 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 97532d2585300ca10ffc4f6d5dda82c5:A 2024-12-04T19:14:15,594 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:15,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 304 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57646 deadline: 1733339715592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:15,594 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:15,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 288 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57632 deadline: 1733339715592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:15,594 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:15,594 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:15,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:15,594 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:15,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:15,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:15,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-04T19:14:15,632 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T19:14:15,632 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T19:14:15,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-12-04T19:14:15,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-04T19:14:15,634 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T19:14:15,634 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T19:14:15,634 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T19:14:15,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-04T19:14:15,786 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-04T19:14:15,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:15,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. as already flushing 2024-12-04T19:14:15,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:15,787 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:15,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:15,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:15,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:15,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 305 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57646 deadline: 1733339715802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:15,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:15,807 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:15,807 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:15,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 289 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57632 deadline: 1733339715802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:15,807 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:15,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:15,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:15,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:15,831 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=1029 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/A/76b3d4c1d4c14371a80b2872183bd32a 2024-12-04T19:14:15,839 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/B/ea16a4fcbbf04d5e9c1089ab16922af7 is 50, key is test_row_0/B:col10/1733339655416/Put/seqid=0 2024-12-04T19:14:15,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741952_1128 (size=12301) 2024-12-04T19:14:15,939 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-04T19:14:15,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:15,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. as already flushing 2024-12-04T19:14:15,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:15,940 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:15,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:15,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:15,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-04T19:14:16,025 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:16,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 248 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57662 deadline: 1733339716022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:16,026 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:16,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 277 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57588 deadline: 1733339716022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:16,026 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 9ba2734cd39f,33969,1733339629791 for 'test_row_2' in 97532d2585300ca10ffc4f6d5dda82c5 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2157 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-04T19:14:16,027 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:16,027 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:16,026 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 9ba2734cd39f,33969,1733339629791 for 'test_row_0' in 97532d2585300ca10ffc4f6d5dda82c5 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2158 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-04T19:14:16,027 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:16,027 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:16,027 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:16,027 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:16,092 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-04T19:14:16,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:16,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. as already flushing 2024-12-04T19:14:16,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:16,093 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:16,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:16,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:16,115 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:16,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 306 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57646 deadline: 1733339716112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:16,116 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:16,116 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:16,116 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:16,116 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:16,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 290 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57632 deadline: 1733339716112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:16,116 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:16,116 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:16,116 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:16,244 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=1029 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/B/ea16a4fcbbf04d5e9c1089ab16922af7 2024-12-04T19:14:16,247 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-04T19:14:16,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:16,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. as already flushing 2024-12-04T19:14:16,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:16,248 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:16,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:16,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:16,255 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/C/955d1e23a37a4362a048d460373a7e57 is 50, key is test_row_0/C:col10/1733339655416/Put/seqid=0 2024-12-04T19:14:16,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-04T19:14:16,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741953_1129 (size=12301) 2024-12-04T19:14:16,377 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-12-04T19:14:16,377 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T19:14:16,379 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-12-04T19:14:16,379 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T19:14:16,381 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-12-04T19:14:16,381 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-12-04T19:14:16,381 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T19:14:16,381 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T19:14:16,382 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:14:16,382 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:14:16,382 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:14:16,382 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:14:16,383 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:14:16,383 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:14:16,383 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:14:16,383 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:14:16,383 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T19:14:16,383 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T19:14:16,383 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T19:14:16,383 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T19:14:16,401 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-04T19:14:16,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:16,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. as already flushing 2024-12-04T19:14:16,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:16,401 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:16,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:16,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:16,556 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-04T19:14:16,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:16,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. as already flushing 2024-12-04T19:14:16,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:16,557 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:16,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:16,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:16,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:16,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 307 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57646 deadline: 1733339716622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:16,624 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:16,624 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:16,624 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:16,633 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:16,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 291 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57632 deadline: 1733339716633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:16,634 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:16,634 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:16,635 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:16,673 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=1029 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/C/955d1e23a37a4362a048d460373a7e57 2024-12-04T19:14:16,682 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/A/76b3d4c1d4c14371a80b2872183bd32a as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/76b3d4c1d4c14371a80b2872183bd32a 2024-12-04T19:14:16,688 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/76b3d4c1d4c14371a80b2872183bd32a, entries=200, sequenceid=1029, filesize=14.4 K 2024-12-04T19:14:16,689 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/B/ea16a4fcbbf04d5e9c1089ab16922af7 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/ea16a4fcbbf04d5e9c1089ab16922af7 2024-12-04T19:14:16,694 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/ea16a4fcbbf04d5e9c1089ab16922af7, entries=150, sequenceid=1029, filesize=12.0 K 2024-12-04T19:14:16,695 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/C/955d1e23a37a4362a048d460373a7e57 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/955d1e23a37a4362a048d460373a7e57 2024-12-04T19:14:16,700 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/955d1e23a37a4362a048d460373a7e57, entries=150, sequenceid=1029, filesize=12.0 K 2024-12-04T19:14:16,701 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 97532d2585300ca10ffc4f6d5dda82c5 in 1281ms, sequenceid=1029, compaction requested=true 2024-12-04T19:14:16,701 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 97532d2585300ca10ffc4f6d5dda82c5: 2024-12-04T19:14:16,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 97532d2585300ca10ffc4f6d5dda82c5:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T19:14:16,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:14:16,701 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T19:14:16,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 97532d2585300ca10ffc4f6d5dda82c5:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T19:14:16,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:14:16,701 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T19:14:16,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 97532d2585300ca10ffc4f6d5dda82c5:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T19:14:16,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T19:14:16,702 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38125 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T19:14:16,702 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35685 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T19:14:16,702 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1541): 97532d2585300ca10ffc4f6d5dda82c5/B is initiating minor compaction (all files) 2024-12-04T19:14:16,702 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1541): 97532d2585300ca10ffc4f6d5dda82c5/A is initiating minor compaction (all files) 2024-12-04T19:14:16,703 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 97532d2585300ca10ffc4f6d5dda82c5/B in TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:16,703 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 97532d2585300ca10ffc4f6d5dda82c5/A in TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:16,703 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/399abbcee1f346b5994fcff6c8baf7dc, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/73d1166548d94b7e9941bb2f0efdffcc, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/ea16a4fcbbf04d5e9c1089ab16922af7] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp, totalSize=34.8 K 2024-12-04T19:14:16,703 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/a0d456cc73b74372a8db3e71edd34ec8, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/0cec263b854d4c918259efbb6a008e04, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/76b3d4c1d4c14371a80b2872183bd32a] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp, totalSize=37.2 K 2024-12-04T19:14:16,703 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting a0d456cc73b74372a8db3e71edd34ec8, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=1012, earliestPutTs=1733339653265 2024-12-04T19:14:16,703 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 399abbcee1f346b5994fcff6c8baf7dc, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=1012, earliestPutTs=1733339653265 2024-12-04T19:14:16,703 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0cec263b854d4c918259efbb6a008e04, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=1019, earliestPutTs=1733339653867 2024-12-04T19:14:16,704 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 73d1166548d94b7e9941bb2f0efdffcc, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=1019, earliestPutTs=1733339653867 2024-12-04T19:14:16,704 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 76b3d4c1d4c14371a80b2872183bd32a, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=1029, earliestPutTs=1733339655413 2024-12-04T19:14:16,704 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting ea16a4fcbbf04d5e9c1089ab16922af7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=1029, earliestPutTs=1733339655416 2024-12-04T19:14:16,711 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 97532d2585300ca10ffc4f6d5dda82c5#B#compaction#118 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:14:16,711 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 97532d2585300ca10ffc4f6d5dda82c5#A#compaction#117 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:14:16,712 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/A/cd986753a1364210826c8a5801fd6ac9 is 50, key is test_row_0/A:col10/1733339655416/Put/seqid=0 2024-12-04T19:14:16,712 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/B/3b342c1237984deb90fd0e70c3db97d8 is 50, key is test_row_0/B:col10/1733339655416/Put/seqid=0 2024-12-04T19:14:16,713 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-04T19:14:16,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:16,713 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing 97532d2585300ca10ffc4f6d5dda82c5 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-12-04T19:14:16,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97532d2585300ca10ffc4f6d5dda82c5, store=A 2024-12-04T19:14:16,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T19:14:16,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97532d2585300ca10ffc4f6d5dda82c5, store=B 2024-12-04T19:14:16,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T19:14:16,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97532d2585300ca10ffc4f6d5dda82c5, store=C 2024-12-04T19:14:16,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T19:14:16,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741954_1130 (size=13629) 2024-12-04T19:14:16,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741955_1131 (size=13629) 2024-12-04T19:14:16,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/A/0246cd1c8a474d5ba466ca69d1930b86 is 50, key is test_row_0/A:col10/1733339655483/Put/seqid=0 2024-12-04T19:14:16,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741956_1132 (size=12301) 2024-12-04T19:14:16,724 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/B/3b342c1237984deb90fd0e70c3db97d8 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/3b342c1237984deb90fd0e70c3db97d8 2024-12-04T19:14:16,728 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 97532d2585300ca10ffc4f6d5dda82c5/B of 97532d2585300ca10ffc4f6d5dda82c5 into 3b342c1237984deb90fd0e70c3db97d8(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:14:16,729 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 97532d2585300ca10ffc4f6d5dda82c5: 2024-12-04T19:14:16,729 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., storeName=97532d2585300ca10ffc4f6d5dda82c5/B, priority=13, startTime=1733339656701; duration=0sec 2024-12-04T19:14:16,729 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T19:14:16,729 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 97532d2585300ca10ffc4f6d5dda82c5:B 2024-12-04T19:14:16,729 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T19:14:16,730 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35685 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T19:14:16,730 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1541): 97532d2585300ca10ffc4f6d5dda82c5/C is initiating minor compaction (all files) 2024-12-04T19:14:16,730 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 97532d2585300ca10ffc4f6d5dda82c5/C in TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:16,730 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/b48c55df66754366bb8ce7687eae8557, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/22a1d9d85b344cccbdff5bc47f47a415, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/955d1e23a37a4362a048d460373a7e57] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp, totalSize=34.8 K 2024-12-04T19:14:16,730 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting b48c55df66754366bb8ce7687eae8557, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=1012, earliestPutTs=1733339653265 2024-12-04T19:14:16,730 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 22a1d9d85b344cccbdff5bc47f47a415, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=1019, earliestPutTs=1733339653867 2024-12-04T19:14:16,731 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 955d1e23a37a4362a048d460373a7e57, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=1029, earliestPutTs=1733339655416 2024-12-04T19:14:16,737 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 97532d2585300ca10ffc4f6d5dda82c5#C#compaction#120 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:14:16,737 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/C/c43097fb61784411afd2a4de1418639c is 50, key is test_row_0/C:col10/1733339655416/Put/seqid=0 2024-12-04T19:14:16,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741957_1133 (size=13629) 2024-12-04T19:14:16,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-04T19:14:17,124 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=1065 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/A/0246cd1c8a474d5ba466ca69d1930b86 2024-12-04T19:14:17,129 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/A/cd986753a1364210826c8a5801fd6ac9 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/cd986753a1364210826c8a5801fd6ac9 2024-12-04T19:14:17,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/B/e491f1dd840f4606b28fbe40bbc2b7a6 is 50, key is test_row_0/B:col10/1733339655483/Put/seqid=0 2024-12-04T19:14:17,136 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 97532d2585300ca10ffc4f6d5dda82c5/A of 97532d2585300ca10ffc4f6d5dda82c5 into cd986753a1364210826c8a5801fd6ac9(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:14:17,136 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 97532d2585300ca10ffc4f6d5dda82c5: 2024-12-04T19:14:17,136 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., storeName=97532d2585300ca10ffc4f6d5dda82c5/A, priority=13, startTime=1733339656701; duration=0sec 2024-12-04T19:14:17,137 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:14:17,137 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 97532d2585300ca10ffc4f6d5dda82c5:A 2024-12-04T19:14:17,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741958_1134 (size=12301) 2024-12-04T19:14:17,146 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/C/c43097fb61784411afd2a4de1418639c as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/c43097fb61784411afd2a4de1418639c 2024-12-04T19:14:17,150 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 97532d2585300ca10ffc4f6d5dda82c5/C of 97532d2585300ca10ffc4f6d5dda82c5 into c43097fb61784411afd2a4de1418639c(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:14:17,150 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 97532d2585300ca10ffc4f6d5dda82c5: 2024-12-04T19:14:17,150 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., storeName=97532d2585300ca10ffc4f6d5dda82c5/C, priority=13, startTime=1733339656701; duration=0sec 2024-12-04T19:14:17,150 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:14:17,150 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 97532d2585300ca10ffc4f6d5dda82c5:C 2024-12-04T19:14:17,540 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=1065 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/B/e491f1dd840f4606b28fbe40bbc2b7a6 2024-12-04T19:14:17,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/C/d5be687405664c01afa238ba0dda22c7 is 50, key is test_row_0/C:col10/1733339655483/Put/seqid=0 2024-12-04T19:14:17,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741959_1135 (size=12301) 2024-12-04T19:14:17,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(8855): Flush requested on 97532d2585300ca10ffc4f6d5dda82c5 2024-12-04T19:14:17,648 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. as already flushing 2024-12-04T19:14:17,648 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-12-04T19:14:17,648 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T19:14:17,648 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:14:17,648 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:14:17,648 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T19:14:17,652 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-12-04T19:14:17,652 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T19:14:17,652 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:14:17,652 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:14:17,652 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T19:14:17,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-04T19:14:17,964 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=1065 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/C/d5be687405664c01afa238ba0dda22c7 2024-12-04T19:14:17,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/A/0246cd1c8a474d5ba466ca69d1930b86 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/0246cd1c8a474d5ba466ca69d1930b86 2024-12-04T19:14:17,977 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/0246cd1c8a474d5ba466ca69d1930b86, entries=150, sequenceid=1065, filesize=12.0 K 2024-12-04T19:14:17,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/B/e491f1dd840f4606b28fbe40bbc2b7a6 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/e491f1dd840f4606b28fbe40bbc2b7a6 2024-12-04T19:14:17,985 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/e491f1dd840f4606b28fbe40bbc2b7a6, entries=150, sequenceid=1065, filesize=12.0 K 2024-12-04T19:14:17,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/C/d5be687405664c01afa238ba0dda22c7 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/d5be687405664c01afa238ba0dda22c7 2024-12-04T19:14:17,993 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/d5be687405664c01afa238ba0dda22c7, entries=150, sequenceid=1065, filesize=12.0 K 2024-12-04T19:14:17,994 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=13.42 KB/13740 for 97532d2585300ca10ffc4f6d5dda82c5 in 1280ms, sequenceid=1065, compaction requested=false 2024-12-04T19:14:17,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for 97532d2585300ca10ffc4f6d5dda82c5: 2024-12-04T19:14:17,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:17,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-12-04T19:14:17,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-12-04T19:14:17,998 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-12-04T19:14:17,998 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3610 sec 2024-12-04T19:14:18,000 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 2.3670 sec 2024-12-04T19:14:18,013 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-04T19:14:18,043 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-12-04T19:14:18,043 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T19:14:18,043 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:14:18,043 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:14:18,043 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T19:14:18,053 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-12-04T19:14:18,053 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T19:14:18,053 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:14:18,053 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:14:18,053 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T19:14:19,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-04T19:14:19,803 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T19:14:24,697 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-12-04T19:14:24,698 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T19:14:24,698 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:14:24,698 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:14:24,698 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T19:14:24,699 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-04T19:14:24,699 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 211 2024-12-04T19:14:24,699 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 227 2024-12-04T19:14:24,699 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 240 2024-12-04T19:14:24,699 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 88 2024-12-04T19:14:24,700 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 183 2024-12-04T19:14:24,700 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-04T19:14:24,700 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3775 2024-12-04T19:14:24,700 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3910 2024-12-04T19:14:24,700 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-04T19:14:24,700 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1734 2024-12-04T19:14:24,701 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5198 rows 2024-12-04T19:14:24,701 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1715 2024-12-04T19:14:24,701 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5142 rows 2024-12-04T19:14:24,702 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-04T19:14:24,702 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T19:14:24,702 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:14:24,703 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:14:24,703 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T19:14:24,709 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-04T19:14:24,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-04T19:14:24,717 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733339664717"}]},"ts":"1733339664717"} 2024-12-04T19:14:24,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-04T19:14:24,719 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-04T19:14:24,719 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-04T19:14:24,720 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-04T19:14:24,724 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=97532d2585300ca10ffc4f6d5dda82c5, UNASSIGN}] 2024-12-04T19:14:24,725 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=97532d2585300ca10ffc4f6d5dda82c5, UNASSIGN 2024-12-04T19:14:24,726 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=97532d2585300ca10ffc4f6d5dda82c5, regionState=CLOSING, regionLocation=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:24,728 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=97532d2585300ca10ffc4f6d5dda82c5, UNASSIGN because future has completed 2024-12-04T19:14:24,728 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-04T19:14:24,729 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; CloseRegionProcedure 97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791}] 2024-12-04T19:14:24,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-04T19:14:24,884 INFO [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(122): Close 97532d2585300ca10ffc4f6d5dda82c5 2024-12-04T19:14:24,885 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-04T19:14:24,885 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1722): Closing 97532d2585300ca10ffc4f6d5dda82c5, disabling compactions & flushes 2024-12-04T19:14:24,885 INFO [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:24,885 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:24,885 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. after waiting 0 ms 2024-12-04T19:14:24,885 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:24,885 INFO [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(2902): Flushing 97532d2585300ca10ffc4f6d5dda82c5 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-04T19:14:24,886 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97532d2585300ca10ffc4f6d5dda82c5, store=A 2024-12-04T19:14:24,886 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:14:24,886 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97532d2585300ca10ffc4f6d5dda82c5, store=B 2024-12-04T19:14:24,886 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:14:24,886 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97532d2585300ca10ffc4f6d5dda82c5, store=C 2024-12-04T19:14:24,886 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:14:24,890 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/A/12ae8baa48794f43b0947268e6b29797 is 50, key is test_row_0/A:col10/1733339658052/Put/seqid=0 2024-12-04T19:14:24,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741960_1136 (size=12301) 2024-12-04T19:14:25,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-04T19:14:25,296 INFO [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=1076 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/A/12ae8baa48794f43b0947268e6b29797 2024-12-04T19:14:25,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-04T19:14:25,361 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/B/c2ed311e23d4409c8bb09d89c1c9354c is 50, key is test_row_0/B:col10/1733339658052/Put/seqid=0 2024-12-04T19:14:25,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741961_1137 (size=12301) 2024-12-04T19:14:25,766 INFO [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=1076 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/B/c2ed311e23d4409c8bb09d89c1c9354c 2024-12-04T19:14:25,783 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/C/d6a04997746a4a08817b7b9a9fc9b4ea is 50, key is test_row_0/C:col10/1733339658052/Put/seqid=0 2024-12-04T19:14:25,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741962_1138 (size=12301) 2024-12-04T19:14:25,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-04T19:14:26,189 INFO [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=1076 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/C/d6a04997746a4a08817b7b9a9fc9b4ea 2024-12-04T19:14:26,201 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/A/12ae8baa48794f43b0947268e6b29797 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/12ae8baa48794f43b0947268e6b29797 2024-12-04T19:14:26,207 INFO [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/12ae8baa48794f43b0947268e6b29797, entries=150, sequenceid=1076, filesize=12.0 K 2024-12-04T19:14:26,208 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/B/c2ed311e23d4409c8bb09d89c1c9354c as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/c2ed311e23d4409c8bb09d89c1c9354c 2024-12-04T19:14:26,214 INFO [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/c2ed311e23d4409c8bb09d89c1c9354c, entries=150, sequenceid=1076, filesize=12.0 K 2024-12-04T19:14:26,215 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/.tmp/C/d6a04997746a4a08817b7b9a9fc9b4ea as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/d6a04997746a4a08817b7b9a9fc9b4ea 2024-12-04T19:14:26,221 INFO [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/d6a04997746a4a08817b7b9a9fc9b4ea, entries=150, sequenceid=1076, filesize=12.0 K 2024-12-04T19:14:26,222 INFO [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 97532d2585300ca10ffc4f6d5dda82c5 in 1337ms, sequenceid=1076, compaction requested=true 2024-12-04T19:14:26,223 DEBUG [StoreCloser-TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/e6ff7769f71744978011ed7aa2a84e79, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/0da65db6ed684622a05bda6392f565b5, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/0c45af55b8624ba189b927011fc9ce88, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/03a3da469b554415b4cdcf6bfcf2fa84, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/8e5c7f579ecf46d2ae3b80c37a741dbb, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/4eec874e412f4284be5c88e2b57498f9, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/574fea5039fc4ce88a7a56c9c5d34f8c, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/8c901c31e3ca40fc811b6c14f3bfe19e, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/d2f5d28c3461429ea2b98166e4dd052a, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/e43a20dbb858412781826acb0d484fa8, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/172a40f262ed4b00a129eb0eccdb9270, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/d4de660048744e9ba0352a0b0efdce22, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/d6c0763bcc63432c9edbc0f9863a9b01, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/6478e3ec41bd4f359a507437ee552614, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/76684f10c0fa46c8b8b51579c8fc26e0, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/aa86364401b543fbb21be4b3a8d8ccff, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/f83dce9ccdb94cfcb9fc06b219ebd06b, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/976ad7e1a2b5451bba298de00d8729e7, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/1df5dc2349e94dfcafc73dd970f302be, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/9521516ec8c5466497ff7a8000a00373, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/6d0f240a5f3b41279b0532d9945f13c2, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/8db49ed06754490b9ddca28e4cf8bdd9, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/deaa1229f50743d6aa1fcbf8dce3e58b, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/8d8364f89bff449c84e8844a9596ea65, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/3a9539b46678439a9b7e170071758c8e, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/3bf21243c456499a86f4b4326c850ff1, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/101112c1a7a142b4a6d27c5dd7df4dc0, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/a9cce18da906429b8643be8d1ce1411f, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/ca95ba3aa611478cb4b5b6d185dbc125, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/9b6de1d4f1bd437e857134700f089e80, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/dadc7fdfe9204f92897e903c250815e0, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/f0cf0c8e174c46ef97aa3950baf03824, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/705b5848e50745529c51b041ef849e14, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/42f8b677132d45488fb6def42902618a, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/9f29bc453a894d05a74de59d3c5ad8a3, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/a0d456cc73b74372a8db3e71edd34ec8, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/dda2e7a3beef4071a8cfa38654269204, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/0cec263b854d4c918259efbb6a008e04, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/76b3d4c1d4c14371a80b2872183bd32a] to archive 2024-12-04T19:14:26,226 DEBUG [StoreCloser-TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-04T19:14:26,231 DEBUG [StoreCloser-TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/e6ff7769f71744978011ed7aa2a84e79 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/e6ff7769f71744978011ed7aa2a84e79 2024-12-04T19:14:26,233 DEBUG [StoreCloser-TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/0da65db6ed684622a05bda6392f565b5 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/0da65db6ed684622a05bda6392f565b5 2024-12-04T19:14:26,234 DEBUG [StoreCloser-TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/0c45af55b8624ba189b927011fc9ce88 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/0c45af55b8624ba189b927011fc9ce88 2024-12-04T19:14:26,235 DEBUG [StoreCloser-TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/03a3da469b554415b4cdcf6bfcf2fa84 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/03a3da469b554415b4cdcf6bfcf2fa84 2024-12-04T19:14:26,237 DEBUG [StoreCloser-TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/8e5c7f579ecf46d2ae3b80c37a741dbb to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/8e5c7f579ecf46d2ae3b80c37a741dbb 2024-12-04T19:14:26,238 DEBUG [StoreCloser-TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/4eec874e412f4284be5c88e2b57498f9 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/4eec874e412f4284be5c88e2b57498f9 2024-12-04T19:14:26,239 DEBUG [StoreCloser-TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/574fea5039fc4ce88a7a56c9c5d34f8c to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/574fea5039fc4ce88a7a56c9c5d34f8c 2024-12-04T19:14:26,240 DEBUG [StoreCloser-TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/8c901c31e3ca40fc811b6c14f3bfe19e to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/8c901c31e3ca40fc811b6c14f3bfe19e 2024-12-04T19:14:26,241 DEBUG [StoreCloser-TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/d2f5d28c3461429ea2b98166e4dd052a to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/d2f5d28c3461429ea2b98166e4dd052a 2024-12-04T19:14:26,242 DEBUG [StoreCloser-TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/e43a20dbb858412781826acb0d484fa8 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/e43a20dbb858412781826acb0d484fa8 2024-12-04T19:14:26,243 DEBUG [StoreCloser-TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/172a40f262ed4b00a129eb0eccdb9270 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/172a40f262ed4b00a129eb0eccdb9270 2024-12-04T19:14:26,244 DEBUG [StoreCloser-TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/d4de660048744e9ba0352a0b0efdce22 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/d4de660048744e9ba0352a0b0efdce22 2024-12-04T19:14:26,245 DEBUG [StoreCloser-TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/d6c0763bcc63432c9edbc0f9863a9b01 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/d6c0763bcc63432c9edbc0f9863a9b01 2024-12-04T19:14:26,247 DEBUG [StoreCloser-TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/6478e3ec41bd4f359a507437ee552614 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/6478e3ec41bd4f359a507437ee552614 2024-12-04T19:14:26,248 DEBUG [StoreCloser-TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/76684f10c0fa46c8b8b51579c8fc26e0 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/76684f10c0fa46c8b8b51579c8fc26e0 2024-12-04T19:14:26,250 DEBUG [StoreCloser-TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/aa86364401b543fbb21be4b3a8d8ccff to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/aa86364401b543fbb21be4b3a8d8ccff 2024-12-04T19:14:26,251 DEBUG [StoreCloser-TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/f83dce9ccdb94cfcb9fc06b219ebd06b to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/f83dce9ccdb94cfcb9fc06b219ebd06b 2024-12-04T19:14:26,252 DEBUG [StoreCloser-TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/976ad7e1a2b5451bba298de00d8729e7 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/976ad7e1a2b5451bba298de00d8729e7 2024-12-04T19:14:26,254 DEBUG [StoreCloser-TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/1df5dc2349e94dfcafc73dd970f302be to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/1df5dc2349e94dfcafc73dd970f302be 2024-12-04T19:14:26,255 DEBUG [StoreCloser-TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/9521516ec8c5466497ff7a8000a00373 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/9521516ec8c5466497ff7a8000a00373 2024-12-04T19:14:26,256 DEBUG [StoreCloser-TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/6d0f240a5f3b41279b0532d9945f13c2 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/6d0f240a5f3b41279b0532d9945f13c2 2024-12-04T19:14:26,257 DEBUG [StoreCloser-TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/8db49ed06754490b9ddca28e4cf8bdd9 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/8db49ed06754490b9ddca28e4cf8bdd9 2024-12-04T19:14:26,259 DEBUG [StoreCloser-TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/deaa1229f50743d6aa1fcbf8dce3e58b to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/deaa1229f50743d6aa1fcbf8dce3e58b 2024-12-04T19:14:26,260 DEBUG [StoreCloser-TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/8d8364f89bff449c84e8844a9596ea65 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/8d8364f89bff449c84e8844a9596ea65 2024-12-04T19:14:26,261 DEBUG [StoreCloser-TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/3a9539b46678439a9b7e170071758c8e to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/3a9539b46678439a9b7e170071758c8e 2024-12-04T19:14:26,262 DEBUG [StoreCloser-TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/3bf21243c456499a86f4b4326c850ff1 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/3bf21243c456499a86f4b4326c850ff1 2024-12-04T19:14:26,263 DEBUG [StoreCloser-TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/101112c1a7a142b4a6d27c5dd7df4dc0 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/101112c1a7a142b4a6d27c5dd7df4dc0 2024-12-04T19:14:26,264 DEBUG [StoreCloser-TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/a9cce18da906429b8643be8d1ce1411f to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/a9cce18da906429b8643be8d1ce1411f 2024-12-04T19:14:26,266 DEBUG [StoreCloser-TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/ca95ba3aa611478cb4b5b6d185dbc125 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/ca95ba3aa611478cb4b5b6d185dbc125 2024-12-04T19:14:26,267 DEBUG [StoreCloser-TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/9b6de1d4f1bd437e857134700f089e80 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/9b6de1d4f1bd437e857134700f089e80 2024-12-04T19:14:26,268 DEBUG [StoreCloser-TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/dadc7fdfe9204f92897e903c250815e0 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/dadc7fdfe9204f92897e903c250815e0 2024-12-04T19:14:26,269 DEBUG [StoreCloser-TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/f0cf0c8e174c46ef97aa3950baf03824 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/f0cf0c8e174c46ef97aa3950baf03824 2024-12-04T19:14:26,270 DEBUG [StoreCloser-TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/705b5848e50745529c51b041ef849e14 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/705b5848e50745529c51b041ef849e14 2024-12-04T19:14:26,271 DEBUG [StoreCloser-TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/42f8b677132d45488fb6def42902618a to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/42f8b677132d45488fb6def42902618a 2024-12-04T19:14:26,273 DEBUG [StoreCloser-TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/9f29bc453a894d05a74de59d3c5ad8a3 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/9f29bc453a894d05a74de59d3c5ad8a3 2024-12-04T19:14:26,274 DEBUG [StoreCloser-TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/a0d456cc73b74372a8db3e71edd34ec8 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/a0d456cc73b74372a8db3e71edd34ec8 2024-12-04T19:14:26,275 DEBUG [StoreCloser-TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/dda2e7a3beef4071a8cfa38654269204 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/dda2e7a3beef4071a8cfa38654269204 2024-12-04T19:14:26,276 DEBUG [StoreCloser-TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/0cec263b854d4c918259efbb6a008e04 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/0cec263b854d4c918259efbb6a008e04 2024-12-04T19:14:26,277 DEBUG [StoreCloser-TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/76b3d4c1d4c14371a80b2872183bd32a to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/76b3d4c1d4c14371a80b2872183bd32a 2024-12-04T19:14:26,289 DEBUG [StoreCloser-TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/6756372be3a4446aa9c78e6d12de05f5, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/954d40e092ec4b1aa44f5760c7adf224, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/5b523375fe0f463cbcae448e4cb1a547, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/cf37ea9a01f44d7f8a51863f27ae80f6, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/b3b4003cb2f44c5cb8ff6b5cefbf088f, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/368192e7f4d24b35ba5878a7d6687385, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/5e9d422ea79b4da2b74148d91f955530, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/cdf6d60e739f4885bb1ae73c49590444, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/026a0339a5094f14a3405e8577132fcb, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/9cf5c91478ed4473aa486d01eec612e3, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/4540577f834e43b88493db7c299d8aa9, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/ac6fe4b8706746e1974dfba27f3a69f6, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/e80aead9744a4b3d9a5ec095205b13ba, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/cb163233217f441684b6853c6005330b, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/98c45a8d3ab044b1840e3ba40f8fd1bf, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/56a67a1194474da0b04d17914bdd6fac, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/030ec87f2c42467895d43ffc0aae16a9, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/588a065d480541b2bb89161455f082a4, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/9b90062db7c3409fb155cf8f517d9859, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/dde2ff0bc8c1446293366320c362f21d, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/118492520cfd4cd08019712385000d1f, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/5fd32190041147ac98c8c06417907416, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/dc7b1da755c549dda6e5438320ce5f4a, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/c8a6ebbe8f83462cbac7d6a9ac7c7a0e, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/444446ab26da42e4a16a368b55b5b43b, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/5129fb9f4cb74c648896c5ea30ea87de, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/e10369cb90964d01b7a37e9d83adc0d4, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/9596d44d850b4f2c95e0f6fa1042fb7f, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/7269dc6260804590b7fedc61b386ae9e, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/6a01c44df0534f7c8020da9ebaf4583b, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/749ffd0cc5b5401f9ce8b3cc42ba8374, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/a49a3a4cf18143b09c35e06b4fede94b, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/c0cff809fe5042d290ca31c505398583, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/66a0a47638dd41bf96de5b6a8d20d1ea, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/3bc71a1eb83d403abf2de17740dd6625, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/399abbcee1f346b5994fcff6c8baf7dc, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/49fc640ac0e54c3d9a996962a26ad92c, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/73d1166548d94b7e9941bb2f0efdffcc, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/ea16a4fcbbf04d5e9c1089ab16922af7] to archive 2024-12-04T19:14:26,290 DEBUG [StoreCloser-TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-04T19:14:26,292 DEBUG [StoreCloser-TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/6756372be3a4446aa9c78e6d12de05f5 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/6756372be3a4446aa9c78e6d12de05f5 2024-12-04T19:14:26,293 DEBUG [StoreCloser-TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/954d40e092ec4b1aa44f5760c7adf224 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/954d40e092ec4b1aa44f5760c7adf224 2024-12-04T19:14:26,294 DEBUG [StoreCloser-TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/5b523375fe0f463cbcae448e4cb1a547 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/5b523375fe0f463cbcae448e4cb1a547 2024-12-04T19:14:26,295 DEBUG [StoreCloser-TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/cf37ea9a01f44d7f8a51863f27ae80f6 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/cf37ea9a01f44d7f8a51863f27ae80f6 2024-12-04T19:14:26,296 DEBUG [StoreCloser-TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/b3b4003cb2f44c5cb8ff6b5cefbf088f to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/b3b4003cb2f44c5cb8ff6b5cefbf088f 2024-12-04T19:14:26,297 DEBUG [StoreCloser-TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/368192e7f4d24b35ba5878a7d6687385 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/368192e7f4d24b35ba5878a7d6687385 2024-12-04T19:14:26,298 DEBUG [StoreCloser-TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/5e9d422ea79b4da2b74148d91f955530 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/5e9d422ea79b4da2b74148d91f955530 2024-12-04T19:14:26,299 DEBUG [StoreCloser-TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/cdf6d60e739f4885bb1ae73c49590444 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/cdf6d60e739f4885bb1ae73c49590444 2024-12-04T19:14:26,300 DEBUG [StoreCloser-TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/026a0339a5094f14a3405e8577132fcb to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/026a0339a5094f14a3405e8577132fcb 2024-12-04T19:14:26,301 DEBUG [StoreCloser-TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/9cf5c91478ed4473aa486d01eec612e3 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/9cf5c91478ed4473aa486d01eec612e3 2024-12-04T19:14:26,302 DEBUG [StoreCloser-TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/4540577f834e43b88493db7c299d8aa9 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/4540577f834e43b88493db7c299d8aa9 2024-12-04T19:14:26,303 DEBUG [StoreCloser-TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/ac6fe4b8706746e1974dfba27f3a69f6 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/ac6fe4b8706746e1974dfba27f3a69f6 2024-12-04T19:14:26,304 DEBUG [StoreCloser-TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/e80aead9744a4b3d9a5ec095205b13ba to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/e80aead9744a4b3d9a5ec095205b13ba 2024-12-04T19:14:26,305 DEBUG [StoreCloser-TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/cb163233217f441684b6853c6005330b to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/cb163233217f441684b6853c6005330b 2024-12-04T19:14:26,306 DEBUG [StoreCloser-TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/98c45a8d3ab044b1840e3ba40f8fd1bf to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/98c45a8d3ab044b1840e3ba40f8fd1bf 2024-12-04T19:14:26,307 DEBUG [StoreCloser-TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/56a67a1194474da0b04d17914bdd6fac to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/56a67a1194474da0b04d17914bdd6fac 2024-12-04T19:14:26,308 DEBUG [StoreCloser-TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/030ec87f2c42467895d43ffc0aae16a9 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/030ec87f2c42467895d43ffc0aae16a9 2024-12-04T19:14:26,309 DEBUG [StoreCloser-TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/588a065d480541b2bb89161455f082a4 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/588a065d480541b2bb89161455f082a4 2024-12-04T19:14:26,310 DEBUG [StoreCloser-TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/9b90062db7c3409fb155cf8f517d9859 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/9b90062db7c3409fb155cf8f517d9859 2024-12-04T19:14:26,311 DEBUG [StoreCloser-TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/dde2ff0bc8c1446293366320c362f21d to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/dde2ff0bc8c1446293366320c362f21d 2024-12-04T19:14:26,312 DEBUG [StoreCloser-TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/118492520cfd4cd08019712385000d1f to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/118492520cfd4cd08019712385000d1f 2024-12-04T19:14:26,313 DEBUG [StoreCloser-TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/5fd32190041147ac98c8c06417907416 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/5fd32190041147ac98c8c06417907416 2024-12-04T19:14:26,313 DEBUG [StoreCloser-TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/dc7b1da755c549dda6e5438320ce5f4a to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/dc7b1da755c549dda6e5438320ce5f4a 2024-12-04T19:14:26,314 DEBUG [StoreCloser-TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/c8a6ebbe8f83462cbac7d6a9ac7c7a0e to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/c8a6ebbe8f83462cbac7d6a9ac7c7a0e 2024-12-04T19:14:26,315 DEBUG [StoreCloser-TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/444446ab26da42e4a16a368b55b5b43b to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/444446ab26da42e4a16a368b55b5b43b 2024-12-04T19:14:26,316 DEBUG [StoreCloser-TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/5129fb9f4cb74c648896c5ea30ea87de to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/5129fb9f4cb74c648896c5ea30ea87de 2024-12-04T19:14:26,317 DEBUG [StoreCloser-TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/e10369cb90964d01b7a37e9d83adc0d4 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/e10369cb90964d01b7a37e9d83adc0d4 2024-12-04T19:14:26,318 DEBUG [StoreCloser-TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/9596d44d850b4f2c95e0f6fa1042fb7f to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/9596d44d850b4f2c95e0f6fa1042fb7f 2024-12-04T19:14:26,319 DEBUG [StoreCloser-TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/7269dc6260804590b7fedc61b386ae9e to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/7269dc6260804590b7fedc61b386ae9e 2024-12-04T19:14:26,320 DEBUG [StoreCloser-TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/6a01c44df0534f7c8020da9ebaf4583b to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/6a01c44df0534f7c8020da9ebaf4583b 2024-12-04T19:14:26,321 DEBUG [StoreCloser-TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/749ffd0cc5b5401f9ce8b3cc42ba8374 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/749ffd0cc5b5401f9ce8b3cc42ba8374 2024-12-04T19:14:26,322 DEBUG [StoreCloser-TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/a49a3a4cf18143b09c35e06b4fede94b to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/a49a3a4cf18143b09c35e06b4fede94b 2024-12-04T19:14:26,323 DEBUG [StoreCloser-TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/c0cff809fe5042d290ca31c505398583 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/c0cff809fe5042d290ca31c505398583 2024-12-04T19:14:26,324 DEBUG [StoreCloser-TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/66a0a47638dd41bf96de5b6a8d20d1ea to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/66a0a47638dd41bf96de5b6a8d20d1ea 2024-12-04T19:14:26,325 DEBUG [StoreCloser-TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/3bc71a1eb83d403abf2de17740dd6625 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/3bc71a1eb83d403abf2de17740dd6625 2024-12-04T19:14:26,326 DEBUG [StoreCloser-TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/399abbcee1f346b5994fcff6c8baf7dc to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/399abbcee1f346b5994fcff6c8baf7dc 2024-12-04T19:14:26,327 DEBUG [StoreCloser-TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/49fc640ac0e54c3d9a996962a26ad92c to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/49fc640ac0e54c3d9a996962a26ad92c 2024-12-04T19:14:26,328 DEBUG [StoreCloser-TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/73d1166548d94b7e9941bb2f0efdffcc to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/73d1166548d94b7e9941bb2f0efdffcc 2024-12-04T19:14:26,330 DEBUG [StoreCloser-TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/ea16a4fcbbf04d5e9c1089ab16922af7 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/ea16a4fcbbf04d5e9c1089ab16922af7 2024-12-04T19:14:26,331 DEBUG [StoreCloser-TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/fb6347ed02a843fabb95ecbad719fb4f, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/7c9469d1b25141b3a123563a45af1c09, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/d5790d0e114f4e3eb796bfb274c528d8, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/4f39b560c9fc4f1e8f366336a570451c, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/8ebee7e51f544217a47e85bf92e8f86a, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/c7ebb8f4c6c24013908f3453a997bb7a, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/522cf64f0cbb4b04897c7560cf637917, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/cdaa2070afb6475b8e0819d95028b3d8, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/a5e5b1605f684d3d92dca4c244c9e255, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/262f7d96d0db4f5fb6160dcc5fd57892, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/34ba4ba4bd474ee4aa115d1dfbf9ed7e, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/d8b911757c3e44fab3228269524eba8d, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/a247866f99ac4593926f9431f157cee9, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/1aa0e4ddd0a8441cbc89c7cdc8e0b590, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/61ad7fe738b848e7b04804dfcbe9e9cb, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/91b5c29479344ed4b73f314844174cea, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/6dce60eb67004a6a8e2126776181ce17, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/e4d4745a4a0f4114837f5a7c89843c12, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/318c88be542a4a7d9bb01773f39cb224, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/f043e3a19a30454096ade83d61ecd7f3, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/1abf2b7267144301b7bb270b731cc19c, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/910f817d1fe545588bd52367c5d1121d, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/a8bcc34de375484ebb76d81bc84cc21e, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/a18415c5b0c0498190f230fac9cc0e80, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/871a6b37337246948d8a18ea9a5699b8, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/e7cbcacc77824f6588365730273267d3, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/a761a6af9e9c4d2e9f373311bdfb7f2c, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/cfc54bad225644bd8208a72fc8a9ff9d, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/42e2fc4952e74da5b14fb810f5f46417, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/9cf126b8ba8142af96f62597a24537e0, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/efbf054ccfa741d8888bcaf8e61f2aad, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/47e80d2ae8694dc2ba4383513c6298fd, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/9034b21a97844625928ee0df60d8cf53, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/c1cdd9270db24764bf0a2988f8f4d8cc, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/64089ceed5234119a0743305f92393f0, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/b48c55df66754366bb8ce7687eae8557, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/6aba5cadef98445c8a233f9ccc4a89fd, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/22a1d9d85b344cccbdff5bc47f47a415, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/955d1e23a37a4362a048d460373a7e57] to archive 2024-12-04T19:14:26,332 DEBUG [StoreCloser-TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-04T19:14:26,334 DEBUG [StoreCloser-TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/fb6347ed02a843fabb95ecbad719fb4f to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/fb6347ed02a843fabb95ecbad719fb4f 2024-12-04T19:14:26,336 DEBUG [StoreCloser-TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/7c9469d1b25141b3a123563a45af1c09 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/7c9469d1b25141b3a123563a45af1c09 2024-12-04T19:14:26,337 DEBUG [StoreCloser-TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/d5790d0e114f4e3eb796bfb274c528d8 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/d5790d0e114f4e3eb796bfb274c528d8 2024-12-04T19:14:26,338 DEBUG [StoreCloser-TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/4f39b560c9fc4f1e8f366336a570451c to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/4f39b560c9fc4f1e8f366336a570451c 2024-12-04T19:14:26,339 DEBUG [StoreCloser-TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/8ebee7e51f544217a47e85bf92e8f86a to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/8ebee7e51f544217a47e85bf92e8f86a 2024-12-04T19:14:26,341 DEBUG [StoreCloser-TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/c7ebb8f4c6c24013908f3453a997bb7a to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/c7ebb8f4c6c24013908f3453a997bb7a 2024-12-04T19:14:26,342 DEBUG [StoreCloser-TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/522cf64f0cbb4b04897c7560cf637917 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/522cf64f0cbb4b04897c7560cf637917 2024-12-04T19:14:26,343 DEBUG [StoreCloser-TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/cdaa2070afb6475b8e0819d95028b3d8 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/cdaa2070afb6475b8e0819d95028b3d8 2024-12-04T19:14:26,344 DEBUG [StoreCloser-TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/a5e5b1605f684d3d92dca4c244c9e255 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/a5e5b1605f684d3d92dca4c244c9e255 2024-12-04T19:14:26,345 DEBUG [StoreCloser-TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/262f7d96d0db4f5fb6160dcc5fd57892 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/262f7d96d0db4f5fb6160dcc5fd57892 2024-12-04T19:14:26,346 DEBUG [StoreCloser-TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/34ba4ba4bd474ee4aa115d1dfbf9ed7e to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/34ba4ba4bd474ee4aa115d1dfbf9ed7e 2024-12-04T19:14:26,347 DEBUG [StoreCloser-TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/d8b911757c3e44fab3228269524eba8d to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/d8b911757c3e44fab3228269524eba8d 2024-12-04T19:14:26,348 DEBUG [StoreCloser-TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/a247866f99ac4593926f9431f157cee9 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/a247866f99ac4593926f9431f157cee9 2024-12-04T19:14:26,349 DEBUG [StoreCloser-TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/1aa0e4ddd0a8441cbc89c7cdc8e0b590 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/1aa0e4ddd0a8441cbc89c7cdc8e0b590 2024-12-04T19:14:26,350 DEBUG [StoreCloser-TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/61ad7fe738b848e7b04804dfcbe9e9cb to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/61ad7fe738b848e7b04804dfcbe9e9cb 2024-12-04T19:14:26,352 DEBUG [StoreCloser-TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/91b5c29479344ed4b73f314844174cea to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/91b5c29479344ed4b73f314844174cea 2024-12-04T19:14:26,353 DEBUG [StoreCloser-TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/6dce60eb67004a6a8e2126776181ce17 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/6dce60eb67004a6a8e2126776181ce17 2024-12-04T19:14:26,354 DEBUG [StoreCloser-TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/e4d4745a4a0f4114837f5a7c89843c12 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/e4d4745a4a0f4114837f5a7c89843c12 2024-12-04T19:14:26,355 DEBUG [StoreCloser-TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/318c88be542a4a7d9bb01773f39cb224 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/318c88be542a4a7d9bb01773f39cb224 2024-12-04T19:14:26,356 DEBUG [StoreCloser-TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/f043e3a19a30454096ade83d61ecd7f3 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/f043e3a19a30454096ade83d61ecd7f3 2024-12-04T19:14:26,357 DEBUG [StoreCloser-TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/1abf2b7267144301b7bb270b731cc19c to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/1abf2b7267144301b7bb270b731cc19c 2024-12-04T19:14:26,358 DEBUG [StoreCloser-TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/910f817d1fe545588bd52367c5d1121d to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/910f817d1fe545588bd52367c5d1121d 2024-12-04T19:14:26,359 DEBUG [StoreCloser-TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/a8bcc34de375484ebb76d81bc84cc21e to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/a8bcc34de375484ebb76d81bc84cc21e 2024-12-04T19:14:26,360 DEBUG [StoreCloser-TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/a18415c5b0c0498190f230fac9cc0e80 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/a18415c5b0c0498190f230fac9cc0e80 2024-12-04T19:14:26,361 DEBUG [StoreCloser-TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/871a6b37337246948d8a18ea9a5699b8 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/871a6b37337246948d8a18ea9a5699b8 2024-12-04T19:14:26,362 DEBUG [StoreCloser-TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/e7cbcacc77824f6588365730273267d3 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/e7cbcacc77824f6588365730273267d3 2024-12-04T19:14:26,363 DEBUG [StoreCloser-TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/a761a6af9e9c4d2e9f373311bdfb7f2c to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/a761a6af9e9c4d2e9f373311bdfb7f2c 2024-12-04T19:14:26,364 DEBUG [StoreCloser-TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/cfc54bad225644bd8208a72fc8a9ff9d to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/cfc54bad225644bd8208a72fc8a9ff9d 2024-12-04T19:14:26,365 DEBUG [StoreCloser-TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/42e2fc4952e74da5b14fb810f5f46417 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/42e2fc4952e74da5b14fb810f5f46417 2024-12-04T19:14:26,366 DEBUG [StoreCloser-TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/9cf126b8ba8142af96f62597a24537e0 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/9cf126b8ba8142af96f62597a24537e0 2024-12-04T19:14:26,367 DEBUG [StoreCloser-TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/efbf054ccfa741d8888bcaf8e61f2aad to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/efbf054ccfa741d8888bcaf8e61f2aad 2024-12-04T19:14:26,368 DEBUG [StoreCloser-TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/47e80d2ae8694dc2ba4383513c6298fd to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/47e80d2ae8694dc2ba4383513c6298fd 2024-12-04T19:14:26,369 DEBUG [StoreCloser-TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/9034b21a97844625928ee0df60d8cf53 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/9034b21a97844625928ee0df60d8cf53 2024-12-04T19:14:26,370 DEBUG [StoreCloser-TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/c1cdd9270db24764bf0a2988f8f4d8cc to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/c1cdd9270db24764bf0a2988f8f4d8cc 2024-12-04T19:14:26,372 DEBUG [StoreCloser-TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/64089ceed5234119a0743305f92393f0 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/64089ceed5234119a0743305f92393f0 2024-12-04T19:14:26,373 DEBUG [StoreCloser-TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/b48c55df66754366bb8ce7687eae8557 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/b48c55df66754366bb8ce7687eae8557 2024-12-04T19:14:26,374 DEBUG [StoreCloser-TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/6aba5cadef98445c8a233f9ccc4a89fd to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/6aba5cadef98445c8a233f9ccc4a89fd 2024-12-04T19:14:26,375 DEBUG [StoreCloser-TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/22a1d9d85b344cccbdff5bc47f47a415 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/22a1d9d85b344cccbdff5bc47f47a415 2024-12-04T19:14:26,376 DEBUG [StoreCloser-TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/955d1e23a37a4362a048d460373a7e57 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/955d1e23a37a4362a048d460373a7e57 2024-12-04T19:14:26,381 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/recovered.edits/1079.seqid, newMaxSeqId=1079, maxSeqId=1 2024-12-04T19:14:26,381 INFO [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. 2024-12-04T19:14:26,381 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1676): Region close journal for 97532d2585300ca10ffc4f6d5dda82c5: Waiting for close lock at 1733339664885Running coprocessor pre-close hooks at 1733339664885Disabling compacts and flushes for region at 1733339664885Disabling writes for close at 1733339664885Obtaining lock to block concurrent updates at 1733339664885Preparing flush snapshotting stores in 97532d2585300ca10ffc4f6d5dda82c5 at 1733339664885Finished memstore snapshotting TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1733339664886 (+1 ms)Flushing stores of TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5. at 1733339664886Flushing 97532d2585300ca10ffc4f6d5dda82c5/A: creating writer at 1733339664887 (+1 ms)Flushing 97532d2585300ca10ffc4f6d5dda82c5/A: appending metadata at 1733339664890 (+3 ms)Flushing 97532d2585300ca10ffc4f6d5dda82c5/A: closing flushed file at 1733339664890Flushing 97532d2585300ca10ffc4f6d5dda82c5/B: creating writer at 1733339665354 (+464 ms)Flushing 97532d2585300ca10ffc4f6d5dda82c5/B: appending metadata at 1733339665360 (+6 ms)Flushing 97532d2585300ca10ffc4f6d5dda82c5/B: closing flushed file at 1733339665360Flushing 97532d2585300ca10ffc4f6d5dda82c5/C: creating writer at 1733339665777 (+417 ms)Flushing 97532d2585300ca10ffc4f6d5dda82c5/C: appending metadata at 1733339665782 (+5 ms)Flushing 97532d2585300ca10ffc4f6d5dda82c5/C: closing flushed file at 1733339665782Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@69e5269e: reopening flushed file at 1733339666200 (+418 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4d17dbd3: reopening flushed file at 1733339666207 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@343fc07e: reopening flushed file at 1733339666214 (+7 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 97532d2585300ca10ffc4f6d5dda82c5 in 1337ms, sequenceid=1076, compaction requested=true at 1733339666222 (+8 ms)Writing region close event to WAL at 1733339666377 (+155 ms)Running coprocessor post-close hooks at 1733339666381 (+4 ms)Closed at 1733339666381 2024-12-04T19:14:26,384 INFO [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(157): Closed 97532d2585300ca10ffc4f6d5dda82c5 2024-12-04T19:14:26,384 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=97532d2585300ca10ffc4f6d5dda82c5, regionState=CLOSED 2024-12-04T19:14:26,386 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=27, state=RUNNABLE, hasLock=false; CloseRegionProcedure 97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 because future has completed 2024-12-04T19:14:26,388 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-12-04T19:14:26,388 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; CloseRegionProcedure 97532d2585300ca10ffc4f6d5dda82c5, server=9ba2734cd39f,33969,1733339629791 in 1.6580 sec 2024-12-04T19:14:26,391 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-12-04T19:14:26,391 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=97532d2585300ca10ffc4f6d5dda82c5, UNASSIGN in 1.6640 sec 2024-12-04T19:14:26,394 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-12-04T19:14:26,394 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.6720 sec 2024-12-04T19:14:26,395 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733339666395"}]},"ts":"1733339666395"} 2024-12-04T19:14:26,397 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-04T19:14:26,397 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-04T19:14:26,398 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.6860 sec 2024-12-04T19:14:26,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-04T19:14:26,862 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-04T19:14:26,864 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-04T19:14:26,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T19:14:26,869 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=29, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T19:14:26,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-04T19:14:26,870 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=29, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T19:14:26,874 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5 2024-12-04T19:14:26,879 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A, FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B, FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C, FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/recovered.edits] 2024-12-04T19:14:26,882 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/0246cd1c8a474d5ba466ca69d1930b86 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/0246cd1c8a474d5ba466ca69d1930b86 2024-12-04T19:14:26,884 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/12ae8baa48794f43b0947268e6b29797 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/12ae8baa48794f43b0947268e6b29797 2024-12-04T19:14:26,885 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/cd986753a1364210826c8a5801fd6ac9 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/A/cd986753a1364210826c8a5801fd6ac9 2024-12-04T19:14:26,890 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/3b342c1237984deb90fd0e70c3db97d8 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/3b342c1237984deb90fd0e70c3db97d8 2024-12-04T19:14:26,892 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/c2ed311e23d4409c8bb09d89c1c9354c to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/c2ed311e23d4409c8bb09d89c1c9354c 2024-12-04T19:14:26,893 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/e491f1dd840f4606b28fbe40bbc2b7a6 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/B/e491f1dd840f4606b28fbe40bbc2b7a6 2024-12-04T19:14:26,896 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/c43097fb61784411afd2a4de1418639c to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/c43097fb61784411afd2a4de1418639c 2024-12-04T19:14:26,898 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/d5be687405664c01afa238ba0dda22c7 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/d5be687405664c01afa238ba0dda22c7 2024-12-04T19:14:26,899 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/d6a04997746a4a08817b7b9a9fc9b4ea to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/C/d6a04997746a4a08817b7b9a9fc9b4ea 2024-12-04T19:14:26,903 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/recovered.edits/1079.seqid to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5/recovered.edits/1079.seqid 2024-12-04T19:14:26,903 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/97532d2585300ca10ffc4f6d5dda82c5 2024-12-04T19:14:26,903 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-04T19:14:26,909 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=29, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T19:14:26,913 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-12-04T19:14:26,918 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-04T19:14:26,948 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-04T19:14:26,949 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=29, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T19:14:26,950 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-04T19:14:26,950 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733339666950"}]},"ts":"9223372036854775807"} 2024-12-04T19:14:26,954 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-04T19:14:26,954 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 97532d2585300ca10ffc4f6d5dda82c5, NAME => 'TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5.', STARTKEY => '', ENDKEY => ''}] 2024-12-04T19:14:26,954 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-04T19:14:26,955 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733339666954"}]},"ts":"9223372036854775807"} 2024-12-04T19:14:26,957 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-04T19:14:26,958 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=29, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T19:14:26,959 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 93 msec 2024-12-04T19:14:26,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-04T19:14:26,982 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-04T19:14:26,982 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-04T19:14:26,995 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=247 (was 211) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;9ba2734cd39f:33969-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1487874761_22 at /127.0.0.1:41752 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=449 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=361 (was 272) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=15206 (was 15782) 2024-12-04T19:14:27,003 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=247, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=361, ProcessCount=11, AvailableMemoryMB=15205 2024-12-04T19:14:27,005 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-04T19:14:27,005 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T19:14:27,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] procedure2.ProcedureExecutor(1139): Stored pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-04T19:14:27,007 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-04T19:14:27,007 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:27,007 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 30 2024-12-04T19:14:27,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-12-04T19:14:27,008 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-04T19:14:27,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741963_1139 (size=1061) 2024-12-04T19:14:27,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-12-04T19:14:27,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-12-04T19:14:27,417 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d 2024-12-04T19:14:27,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741964_1140 (size=53) 2024-12-04T19:14:27,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-12-04T19:14:27,828 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T19:14:27,828 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 6fbcb032a86ef6909d8f2378183195f0, disabling compactions & flushes 2024-12-04T19:14:27,828 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:27,828 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:27,829 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. after waiting 1 ms 2024-12-04T19:14:27,829 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:27,829 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:27,829 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 6fbcb032a86ef6909d8f2378183195f0: Waiting for close lock at 1733339667828Disabling compacts and flushes for region at 1733339667828Disabling writes for close at 1733339667829 (+1 ms)Writing region close event to WAL at 1733339667829Closed at 1733339667829 2024-12-04T19:14:27,830 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-04T19:14:27,830 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733339667830"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733339667830"}]},"ts":"1733339667830"} 2024-12-04T19:14:27,832 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-04T19:14:27,833 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-04T19:14:27,834 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733339667833"}]},"ts":"1733339667833"} 2024-12-04T19:14:27,835 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-04T19:14:27,836 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6fbcb032a86ef6909d8f2378183195f0, ASSIGN}] 2024-12-04T19:14:27,837 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6fbcb032a86ef6909d8f2378183195f0, ASSIGN 2024-12-04T19:14:27,838 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6fbcb032a86ef6909d8f2378183195f0, ASSIGN; state=OFFLINE, location=9ba2734cd39f,33969,1733339629791; forceNewPlan=false, retain=false 2024-12-04T19:14:27,989 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=6fbcb032a86ef6909d8f2378183195f0, regionState=OPENING, regionLocation=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:27,994 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6fbcb032a86ef6909d8f2378183195f0, ASSIGN because future has completed 2024-12-04T19:14:27,995 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791}] 2024-12-04T19:14:28,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-12-04T19:14:28,152 INFO [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:28,152 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7752): Opening region: {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} 2024-12-04T19:14:28,153 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 6fbcb032a86ef6909d8f2378183195f0 2024-12-04T19:14:28,153 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T19:14:28,153 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7794): checking encryption for 6fbcb032a86ef6909d8f2378183195f0 2024-12-04T19:14:28,153 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7797): checking classloading for 6fbcb032a86ef6909d8f2378183195f0 2024-12-04T19:14:28,155 INFO [StoreOpener-6fbcb032a86ef6909d8f2378183195f0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 6fbcb032a86ef6909d8f2378183195f0 2024-12-04T19:14:28,157 INFO [StoreOpener-6fbcb032a86ef6909d8f2378183195f0-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-04T19:14:28,159 INFO [StoreOpener-6fbcb032a86ef6909d8f2378183195f0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6fbcb032a86ef6909d8f2378183195f0 columnFamilyName A 2024-12-04T19:14:28,159 DEBUG [StoreOpener-6fbcb032a86ef6909d8f2378183195f0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:28,160 INFO [StoreOpener-6fbcb032a86ef6909d8f2378183195f0-1 {}] regionserver.HStore(327): Store=6fbcb032a86ef6909d8f2378183195f0/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:14:28,160 INFO [StoreOpener-6fbcb032a86ef6909d8f2378183195f0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 6fbcb032a86ef6909d8f2378183195f0 2024-12-04T19:14:28,162 INFO [StoreOpener-6fbcb032a86ef6909d8f2378183195f0-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-04T19:14:28,162 INFO [StoreOpener-6fbcb032a86ef6909d8f2378183195f0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6fbcb032a86ef6909d8f2378183195f0 columnFamilyName B 2024-12-04T19:14:28,163 DEBUG [StoreOpener-6fbcb032a86ef6909d8f2378183195f0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:28,163 INFO [StoreOpener-6fbcb032a86ef6909d8f2378183195f0-1 {}] regionserver.HStore(327): Store=6fbcb032a86ef6909d8f2378183195f0/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:14:28,163 INFO [StoreOpener-6fbcb032a86ef6909d8f2378183195f0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 6fbcb032a86ef6909d8f2378183195f0 2024-12-04T19:14:28,164 INFO [StoreOpener-6fbcb032a86ef6909d8f2378183195f0-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-04T19:14:28,164 INFO [StoreOpener-6fbcb032a86ef6909d8f2378183195f0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6fbcb032a86ef6909d8f2378183195f0 columnFamilyName C 2024-12-04T19:14:28,164 DEBUG [StoreOpener-6fbcb032a86ef6909d8f2378183195f0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:28,165 INFO [StoreOpener-6fbcb032a86ef6909d8f2378183195f0-1 {}] regionserver.HStore(327): Store=6fbcb032a86ef6909d8f2378183195f0/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:14:28,165 INFO [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:28,165 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1038): replaying wal for 6fbcb032a86ef6909d8f2378183195f0 2024-12-04T19:14:28,166 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0 2024-12-04T19:14:28,166 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0 2024-12-04T19:14:28,167 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1048): stopping wal replay for 6fbcb032a86ef6909d8f2378183195f0 2024-12-04T19:14:28,167 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1060): Cleaning up temporary data for 6fbcb032a86ef6909d8f2378183195f0 2024-12-04T19:14:28,167 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-04T19:14:28,168 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1093): writing seq id for 6fbcb032a86ef6909d8f2378183195f0 2024-12-04T19:14:28,170 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T19:14:28,171 INFO [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1114): Opened 6fbcb032a86ef6909d8f2378183195f0; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63206876, jitterRate=-0.05814415216445923}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-04T19:14:28,171 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6fbcb032a86ef6909d8f2378183195f0 2024-12-04T19:14:28,171 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1006): Region open journal for 6fbcb032a86ef6909d8f2378183195f0: Running coprocessor pre-open hook at 1733339668154Writing region info on filesystem at 1733339668154Initializing all the Stores at 1733339668155 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733339668155Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733339668155Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733339668155Cleaning up temporary data from old regions at 1733339668167 (+12 ms)Running coprocessor post-open hooks at 1733339668171 (+4 ms)Region opened successfully at 1733339668171 2024-12-04T19:14:28,172 INFO [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., pid=32, masterSystemTime=1733339668148 2024-12-04T19:14:28,175 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:28,175 INFO [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:28,176 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=6fbcb032a86ef6909d8f2378183195f0, regionState=OPEN, openSeqNum=2, regionLocation=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:28,177 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 because future has completed 2024-12-04T19:14:28,180 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-12-04T19:14:28,180 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; OpenRegionProcedure 6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 in 183 msec 2024-12-04T19:14:28,182 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-12-04T19:14:28,182 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6fbcb032a86ef6909d8f2378183195f0, ASSIGN in 344 msec 2024-12-04T19:14:28,183 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-04T19:14:28,183 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733339668183"}]},"ts":"1733339668183"} 2024-12-04T19:14:28,185 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-04T19:14:28,186 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-04T19:14:28,187 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1810 sec 2024-12-04T19:14:29,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-12-04T19:14:29,154 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-04T19:14:29,156 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c2bb5eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:14:29,156 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ba2734cd39f,46667,-1 for getting cluster id 2024-12-04T19:14:29,157 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T19:14:29,160 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9baaea35-8dd3-4653-9b7c-03e5b792e644' 2024-12-04T19:14:29,161 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T19:14:29,161 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9baaea35-8dd3-4653-9b7c-03e5b792e644" 2024-12-04T19:14:29,161 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d98e5aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:14:29,162 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ba2734cd39f,46667,-1] 2024-12-04T19:14:29,162 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T19:14:29,163 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:14:29,164 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39372, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T19:14:29,165 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6cedc1ea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:14:29,166 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T19:14:29,166 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ba2734cd39f,33969,1733339629791, seqNum=-1] 2024-12-04T19:14:29,167 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T19:14:29,168 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52062, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T19:14:29,170 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-04T19:14:29,170 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 9ba2734cd39f,46667,1733339628877 2024-12-04T19:14:29,170 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1601abae 2024-12-04T19:14:29,171 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-04T19:14:29,172 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39386, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-04T19:14:29,178 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-04T19:14:29,178 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T19:14:29,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] procedure2.ProcedureExecutor(1139): Stored pid=33, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-04T19:14:29,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741965_1141 (size=1097) 2024-12-04T19:14:29,597 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-04T19:14:29,597 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-04T19:14:29,602 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-04T19:14:29,611 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6fbcb032a86ef6909d8f2378183195f0, REOPEN/MOVE}] 2024-12-04T19:14:29,611 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6fbcb032a86ef6909d8f2378183195f0, REOPEN/MOVE 2024-12-04T19:14:29,612 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=6fbcb032a86ef6909d8f2378183195f0, regionState=CLOSING, regionLocation=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:29,613 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6fbcb032a86ef6909d8f2378183195f0, REOPEN/MOVE because future has completed 2024-12-04T19:14:29,614 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-04T19:14:29,614 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791}] 2024-12-04T19:14:29,767 INFO [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(122): Close 6fbcb032a86ef6909d8f2378183195f0 2024-12-04T19:14:29,767 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-04T19:14:29,767 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1722): Closing 6fbcb032a86ef6909d8f2378183195f0, disabling compactions & flushes 2024-12-04T19:14:29,767 INFO [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:29,767 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:29,767 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. after waiting 0 ms 2024-12-04T19:14:29,767 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:29,774 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-04T19:14:29,775 INFO [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:29,775 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1676): Region close journal for 6fbcb032a86ef6909d8f2378183195f0: Waiting for close lock at 1733339669767Running coprocessor pre-close hooks at 1733339669767Disabling compacts and flushes for region at 1733339669767Disabling writes for close at 1733339669767Writing region close event to WAL at 1733339669768 (+1 ms)Running coprocessor post-close hooks at 1733339669775 (+7 ms)Closed at 1733339669775 2024-12-04T19:14:29,775 WARN [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegionServer(3299): Not adding moved region record: 6fbcb032a86ef6909d8f2378183195f0 to self. 2024-12-04T19:14:29,777 INFO [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(157): Closed 6fbcb032a86ef6909d8f2378183195f0 2024-12-04T19:14:29,777 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=6fbcb032a86ef6909d8f2378183195f0, regionState=CLOSED 2024-12-04T19:14:29,778 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 because future has completed 2024-12-04T19:14:29,780 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-12-04T19:14:29,781 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure 6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 in 165 msec 2024-12-04T19:14:29,781 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6fbcb032a86ef6909d8f2378183195f0, REOPEN/MOVE; state=CLOSED, location=9ba2734cd39f,33969,1733339629791; forceNewPlan=false, retain=true 2024-12-04T19:14:29,932 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=6fbcb032a86ef6909d8f2378183195f0, regionState=OPENING, regionLocation=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:29,933 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6fbcb032a86ef6909d8f2378183195f0, REOPEN/MOVE because future has completed 2024-12-04T19:14:29,934 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791}] 2024-12-04T19:14:30,091 INFO [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:30,091 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7752): Opening region: {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} 2024-12-04T19:14:30,092 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 6fbcb032a86ef6909d8f2378183195f0 2024-12-04T19:14:30,092 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T19:14:30,093 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7794): checking encryption for 6fbcb032a86ef6909d8f2378183195f0 2024-12-04T19:14:30,093 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7797): checking classloading for 6fbcb032a86ef6909d8f2378183195f0 2024-12-04T19:14:30,099 INFO [StoreOpener-6fbcb032a86ef6909d8f2378183195f0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 6fbcb032a86ef6909d8f2378183195f0 2024-12-04T19:14:30,100 INFO [StoreOpener-6fbcb032a86ef6909d8f2378183195f0-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-04T19:14:30,108 INFO [StoreOpener-6fbcb032a86ef6909d8f2378183195f0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6fbcb032a86ef6909d8f2378183195f0 columnFamilyName A 2024-12-04T19:14:30,110 DEBUG [StoreOpener-6fbcb032a86ef6909d8f2378183195f0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:30,111 INFO [StoreOpener-6fbcb032a86ef6909d8f2378183195f0-1 {}] regionserver.HStore(327): Store=6fbcb032a86ef6909d8f2378183195f0/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:14:30,111 INFO [StoreOpener-6fbcb032a86ef6909d8f2378183195f0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 6fbcb032a86ef6909d8f2378183195f0 2024-12-04T19:14:30,112 INFO [StoreOpener-6fbcb032a86ef6909d8f2378183195f0-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-04T19:14:30,112 INFO [StoreOpener-6fbcb032a86ef6909d8f2378183195f0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6fbcb032a86ef6909d8f2378183195f0 columnFamilyName B 2024-12-04T19:14:30,112 DEBUG [StoreOpener-6fbcb032a86ef6909d8f2378183195f0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:30,112 INFO [StoreOpener-6fbcb032a86ef6909d8f2378183195f0-1 {}] regionserver.HStore(327): Store=6fbcb032a86ef6909d8f2378183195f0/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:14:30,112 INFO [StoreOpener-6fbcb032a86ef6909d8f2378183195f0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 6fbcb032a86ef6909d8f2378183195f0 2024-12-04T19:14:30,113 INFO [StoreOpener-6fbcb032a86ef6909d8f2378183195f0-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-04T19:14:30,113 INFO [StoreOpener-6fbcb032a86ef6909d8f2378183195f0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6fbcb032a86ef6909d8f2378183195f0 columnFamilyName C 2024-12-04T19:14:30,113 DEBUG [StoreOpener-6fbcb032a86ef6909d8f2378183195f0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:30,113 INFO [StoreOpener-6fbcb032a86ef6909d8f2378183195f0-1 {}] regionserver.HStore(327): Store=6fbcb032a86ef6909d8f2378183195f0/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:14:30,114 INFO [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:30,114 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1038): replaying wal for 6fbcb032a86ef6909d8f2378183195f0 2024-12-04T19:14:30,114 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0 2024-12-04T19:14:30,116 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0 2024-12-04T19:14:30,117 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1048): stopping wal replay for 6fbcb032a86ef6909d8f2378183195f0 2024-12-04T19:14:30,117 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1060): Cleaning up temporary data for 6fbcb032a86ef6909d8f2378183195f0 2024-12-04T19:14:30,117 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-04T19:14:30,119 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1093): writing seq id for 6fbcb032a86ef6909d8f2378183195f0 2024-12-04T19:14:30,119 INFO [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1114): Opened 6fbcb032a86ef6909d8f2378183195f0; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71326512, jitterRate=0.06284785270690918}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-04T19:14:30,119 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6fbcb032a86ef6909d8f2378183195f0 2024-12-04T19:14:30,121 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1006): Region open journal for 6fbcb032a86ef6909d8f2378183195f0: Running coprocessor pre-open hook at 1733339670093Writing region info on filesystem at 1733339670093Initializing all the Stores at 1733339670095 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733339670095Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733339670095Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733339670095Cleaning up temporary data from old regions at 1733339670117 (+22 ms)Running coprocessor post-open hooks at 1733339670119 (+2 ms)Region opened successfully at 1733339670121 (+2 ms) 2024-12-04T19:14:30,122 INFO [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., pid=37, masterSystemTime=1733339670085 2024-12-04T19:14:30,124 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:30,124 INFO [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:30,125 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=6fbcb032a86ef6909d8f2378183195f0, regionState=OPEN, openSeqNum=5, regionLocation=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:30,126 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 because future has completed 2024-12-04T19:14:30,129 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=35 2024-12-04T19:14:30,129 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=35, state=SUCCESS, hasLock=false; OpenRegionProcedure 6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 in 193 msec 2024-12-04T19:14:30,130 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-12-04T19:14:30,130 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6fbcb032a86ef6909d8f2378183195f0, REOPEN/MOVE in 518 msec 2024-12-04T19:14:30,133 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-12-04T19:14:30,133 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 530 msec 2024-12-04T19:14:30,136 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 953 msec 2024-12-04T19:14:30,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-12-04T19:14:30,138 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-04T19:14:30,139 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e01494b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:14:30,139 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ba2734cd39f,46667,-1 for getting cluster id 2024-12-04T19:14:30,139 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T19:14:30,140 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9baaea35-8dd3-4653-9b7c-03e5b792e644' 2024-12-04T19:14:30,140 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T19:14:30,140 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9baaea35-8dd3-4653-9b7c-03e5b792e644" 2024-12-04T19:14:30,140 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34a70777, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:14:30,140 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ba2734cd39f,46667,-1] 2024-12-04T19:14:30,141 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T19:14:30,141 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:14:30,142 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39392, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T19:14:30,142 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58d1d783, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:14:30,143 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28944b13, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:14:30,143 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ba2734cd39f,46667,-1 for getting cluster id 2024-12-04T19:14:30,143 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T19:14:30,144 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9baaea35-8dd3-4653-9b7c-03e5b792e644' 2024-12-04T19:14:30,144 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T19:14:30,144 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9baaea35-8dd3-4653-9b7c-03e5b792e644" 2024-12-04T19:14:30,144 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5efd2437, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:14:30,144 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ba2734cd39f,46667,-1] 2024-12-04T19:14:30,144 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T19:14:30,144 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:14:30,145 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39406, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T19:14:30,145 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2567db26, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:14:30,146 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a95d199, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:14:30,146 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ba2734cd39f,46667,-1 for getting cluster id 2024-12-04T19:14:30,146 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T19:14:30,147 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9baaea35-8dd3-4653-9b7c-03e5b792e644' 2024-12-04T19:14:30,147 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T19:14:30,147 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9baaea35-8dd3-4653-9b7c-03e5b792e644" 2024-12-04T19:14:30,147 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b28326c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:14:30,148 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ba2734cd39f,46667,-1] 2024-12-04T19:14:30,148 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T19:14:30,148 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:14:30,148 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39420, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T19:14:30,149 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c2b93a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:14:30,150 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e51468f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:14:30,150 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ba2734cd39f,46667,-1 for getting cluster id 2024-12-04T19:14:30,150 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T19:14:30,151 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9baaea35-8dd3-4653-9b7c-03e5b792e644' 2024-12-04T19:14:30,151 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T19:14:30,151 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9baaea35-8dd3-4653-9b7c-03e5b792e644" 2024-12-04T19:14:30,151 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65b3b785, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:14:30,151 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ba2734cd39f,46667,-1] 2024-12-04T19:14:30,151 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T19:14:30,151 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:14:30,152 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39452, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T19:14:30,152 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@425f0b1a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:14:30,153 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a1fcab9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:14:30,153 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ba2734cd39f,46667,-1 for getting cluster id 2024-12-04T19:14:30,153 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T19:14:30,154 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9baaea35-8dd3-4653-9b7c-03e5b792e644' 2024-12-04T19:14:30,154 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T19:14:30,154 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9baaea35-8dd3-4653-9b7c-03e5b792e644" 2024-12-04T19:14:30,154 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6155b81f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:14:30,154 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ba2734cd39f,46667,-1] 2024-12-04T19:14:30,154 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T19:14:30,154 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:14:30,155 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39480, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T19:14:30,156 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78531dc1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:14:30,157 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@337f1b21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:14:30,157 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ba2734cd39f,46667,-1 for getting cluster id 2024-12-04T19:14:30,157 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T19:14:30,157 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9baaea35-8dd3-4653-9b7c-03e5b792e644' 2024-12-04T19:14:30,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T19:14:30,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9baaea35-8dd3-4653-9b7c-03e5b792e644" 2024-12-04T19:14:30,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f5e5f7c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:14:30,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ba2734cd39f,46667,-1] 2024-12-04T19:14:30,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T19:14:30,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:14:30,159 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39492, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T19:14:30,159 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c665522, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:14:30,160 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62885e3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:14:30,160 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ba2734cd39f,46667,-1 for getting cluster id 2024-12-04T19:14:30,160 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T19:14:30,161 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9baaea35-8dd3-4653-9b7c-03e5b792e644' 2024-12-04T19:14:30,161 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T19:14:30,161 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9baaea35-8dd3-4653-9b7c-03e5b792e644" 2024-12-04T19:14:30,161 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5308acec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:14:30,161 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ba2734cd39f,46667,-1] 2024-12-04T19:14:30,161 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T19:14:30,161 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:14:30,162 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39512, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T19:14:30,162 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4746d67b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:14:30,163 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@138dc9c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:14:30,163 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ba2734cd39f,46667,-1 for getting cluster id 2024-12-04T19:14:30,163 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T19:14:30,164 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9baaea35-8dd3-4653-9b7c-03e5b792e644' 2024-12-04T19:14:30,164 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T19:14:30,164 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9baaea35-8dd3-4653-9b7c-03e5b792e644" 2024-12-04T19:14:30,164 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bfb8bf6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:14:30,164 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ba2734cd39f,46667,-1] 2024-12-04T19:14:30,164 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T19:14:30,164 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:14:30,165 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39532, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T19:14:30,166 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7825eb66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:14:30,166 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c264dec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:14:30,166 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ba2734cd39f,46667,-1 for getting cluster id 2024-12-04T19:14:30,166 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T19:14:30,167 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9baaea35-8dd3-4653-9b7c-03e5b792e644' 2024-12-04T19:14:30,167 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T19:14:30,167 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9baaea35-8dd3-4653-9b7c-03e5b792e644" 2024-12-04T19:14:30,167 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75b87027, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:14:30,167 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ba2734cd39f,46667,-1] 2024-12-04T19:14:30,167 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T19:14:30,167 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:14:30,168 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39552, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T19:14:30,169 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cfa90e3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:14:30,169 DEBUG [Thread-696 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T19:14:30,169 DEBUG [Thread-709 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T19:14:30,170 DEBUG [Thread-713 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T19:14:30,170 DEBUG [Thread-700 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T19:14:30,170 DEBUG [Thread-704 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T19:14:30,170 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T19:14:30,170 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ba2734cd39f,33969,1733339629791, seqNum=-1] 2024-12-04T19:14:30,170 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ba2734cd39f,33969,1733339629791, seqNum=-1] 2024-12-04T19:14:30,170 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ba2734cd39f,33969,1733339629791, seqNum=-1] 2024-12-04T19:14:30,171 DEBUG [Thread-696 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ba2734cd39f,33969,1733339629791, seqNum=-1] 2024-12-04T19:14:30,171 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T19:14:30,171 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T19:14:30,171 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T19:14:30,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ba2734cd39f,33969,1733339629791, seqNum=-1] 2024-12-04T19:14:30,171 DEBUG [Thread-711 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T19:14:30,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] procedure2.ProcedureExecutor(1139): Stored pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees 2024-12-04T19:14:30,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T19:14:30,172 DEBUG [Thread-696 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T19:14:30,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-04T19:14:30,172 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52088, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T19:14:30,172 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52078, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T19:14:30,173 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52090, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T19:14:30,173 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T19:14:30,173 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52066, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T19:14:30,173 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ba2734cd39f,33969,1733339629791, seqNum=-1] 2024-12-04T19:14:30,173 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52102, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T19:14:30,173 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T19:14:30,174 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T19:14:30,174 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T19:14:30,174 DEBUG [Thread-707 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T19:14:30,174 DEBUG [Thread-698 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T19:14:30,175 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52110, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T19:14:30,175 DEBUG [Thread-702 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T19:14:30,175 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ba2734cd39f,33969,1733339629791, seqNum=-1] 2024-12-04T19:14:30,175 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T19:14:30,175 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ba2734cd39f,33969,1733339629791, seqNum=-1] 2024-12-04T19:14:30,176 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T19:14:30,177 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ba2734cd39f,33969,1733339629791, seqNum=-1] 2024-12-04T19:14:30,177 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52122, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T19:14:30,177 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52124, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T19:14:30,177 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T19:14:30,177 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5] 2024-12-04T19:14:30,178 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5] 2024-12-04T19:14:30,178 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52134, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T19:14:30,179 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5] 2024-12-04T19:14:30,179 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5] 2024-12-04T19:14:30,180 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5] 2024-12-04T19:14:30,181 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5] 2024-12-04T19:14:30,181 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5] 2024-12-04T19:14:30,181 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5] 2024-12-04T19:14:30,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5] 2024-12-04T19:14:30,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(8855): Flush requested on 6fbcb032a86ef6909d8f2378183195f0 2024-12-04T19:14:30,189 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6fbcb032a86ef6909d8f2378183195f0 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-04T19:14:30,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6fbcb032a86ef6909d8f2378183195f0, store=A 2024-12-04T19:14:30,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:14:30,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6fbcb032a86ef6909d8f2378183195f0, store=B 2024-12-04T19:14:30,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:14:30,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6fbcb032a86ef6909d8f2378183195f0, store=C 2024-12-04T19:14:30,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:14:30,244 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204e34d6008a0c240ae8e719c8f44ddf206_6fbcb032a86ef6909d8f2378183195f0 is 50, key is test_row_0/A:col10/1733339670187/Put/seqid=0 2024-12-04T19:14:30,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741966_1142 (size=14594) 2024-12-04T19:14:30,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-04T19:14:30,289 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733339670261, max timestamp=1733339670284Num uniques 859; 2024-12-04T19:14:30,290 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-04T19:14:30,290 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733339670261, max timestamp=1733339670284Num uniques 859; 2024-12-04T19:14:30,290 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-04T19:14:30,292 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733339670265, max timestamp=1733339670289Num uniques 859; 2024-12-04T19:14:30,293 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T19:14:30,327 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-04T19:14:30,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:30,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. as already flushing 2024-12-04T19:14:30,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:30,327 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:30,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:30,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:30,332 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:30,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52122 deadline: 1733339730318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:30,333 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:30,333 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:30,333 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:30,333 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:30,333 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:30,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52066 deadline: 1733339730321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:30,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52124 deadline: 1733339730323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:30,333 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:30,333 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:30,334 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:30,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52088 deadline: 1733339730326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:30,334 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:30,334 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:30,334 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:30,334 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:30,334 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:30,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52102 deadline: 1733339730329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:30,334 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:30,335 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:30,335 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:30,335 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:30,335 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:30,335 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:30,445 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:30,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52122 deadline: 1733339730442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:30,446 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:30,446 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:30,446 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:30,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52124 deadline: 1733339730442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:30,446 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:30,446 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:30,446 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:30,446 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:30,446 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:30,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52066 deadline: 1733339730443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:30,447 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:30,447 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:30,447 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:30,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:30,447 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:30,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52102 deadline: 1733339730443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:30,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52088 deadline: 1733339730443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:30,448 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:30,448 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:30,448 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:30,448 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:30,449 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:30,449 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:30,482 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-04T19:14:30,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:30,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. as already flushing 2024-12-04T19:14:30,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:30,483 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:30,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:30,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:30,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-04T19:14:30,637 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-04T19:14:30,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:30,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. as already flushing 2024-12-04T19:14:30,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:30,637 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:30,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:30,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:30,655 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:30,655 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:30,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52124 deadline: 1733339730653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:30,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52088 deadline: 1733339730653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:30,656 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:30,656 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:30,656 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:30,656 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:30,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52102 deadline: 1733339730653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:30,656 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:30,656 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:30,656 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:30,656 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:30,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52122 deadline: 1733339730653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:30,657 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:30,657 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:30,657 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:30,657 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:30,657 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:30,657 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:30,657 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:30,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52066 deadline: 1733339730653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:30,658 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:30,658 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:30,658 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:30,679 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:30,686 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204e34d6008a0c240ae8e719c8f44ddf206_6fbcb032a86ef6909d8f2378183195f0 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204e34d6008a0c240ae8e719c8f44ddf206_6fbcb032a86ef6909d8f2378183195f0 2024-12-04T19:14:30,687 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/A/f1de4ce9a5da4ec5b0f7c61e4d79a706, store: [table=TestAcidGuarantees family=A region=6fbcb032a86ef6909d8f2378183195f0] 2024-12-04T19:14:30,694 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/A/f1de4ce9a5da4ec5b0f7c61e4d79a706 is 175, key is test_row_0/A:col10/1733339670187/Put/seqid=0 2024-12-04T19:14:30,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741967_1143 (size=39549) 2024-12-04T19:14:30,711 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/A/f1de4ce9a5da4ec5b0f7c61e4d79a706 2024-12-04T19:14:30,769 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/B/e83d0ddf343c45fca0441609aff38cf5 is 50, key is test_row_0/B:col10/1733339670187/Put/seqid=0 2024-12-04T19:14:30,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741968_1144 (size=12001) 2024-12-04T19:14:30,783 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/B/e83d0ddf343c45fca0441609aff38cf5 2024-12-04T19:14:30,791 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-04T19:14:30,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:30,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. as already flushing 2024-12-04T19:14:30,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:30,792 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:30,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:30,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:30,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-04T19:14:30,814 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/C/e51ba5f5037348e4afce3ad466fae247 is 50, key is test_row_0/C:col10/1733339670187/Put/seqid=0 2024-12-04T19:14:30,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741969_1145 (size=12001) 2024-12-04T19:14:30,950 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-04T19:14:30,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:30,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. as already flushing 2024-12-04T19:14:30,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:30,951 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:30,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:30,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:30,962 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:30,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52124 deadline: 1733339730962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:30,963 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:30,963 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:30,963 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:30,963 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:30,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52102 deadline: 1733339730963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:30,964 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:30,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:30,964 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:30,964 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:30,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52088 deadline: 1733339730963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:30,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:30,965 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:30,965 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:30,965 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:30,965 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:30,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52066 deadline: 1733339730963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:30,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52122 deadline: 1733339730963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:30,966 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:30,966 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:30,966 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:30,966 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:30,966 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:30,966 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:31,106 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-04T19:14:31,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:31,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. as already flushing 2024-12-04T19:14:31,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:31,107 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:31,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:31,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:31,244 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/C/e51ba5f5037348e4afce3ad466fae247 2024-12-04T19:14:31,251 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/A/f1de4ce9a5da4ec5b0f7c61e4d79a706 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/f1de4ce9a5da4ec5b0f7c61e4d79a706 2024-12-04T19:14:31,261 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-04T19:14:31,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:31,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. as already flushing 2024-12-04T19:14:31,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:31,261 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:31,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:31,262 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/f1de4ce9a5da4ec5b0f7c61e4d79a706, entries=200, sequenceid=14, filesize=38.6 K 2024-12-04T19:14:31,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:31,266 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/B/e83d0ddf343c45fca0441609aff38cf5 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/e83d0ddf343c45fca0441609aff38cf5 2024-12-04T19:14:31,277 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/e83d0ddf343c45fca0441609aff38cf5, entries=150, sequenceid=14, filesize=11.7 K 2024-12-04T19:14:31,279 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/C/e51ba5f5037348e4afce3ad466fae247 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/e51ba5f5037348e4afce3ad466fae247 2024-12-04T19:14:31,297 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/e51ba5f5037348e4afce3ad466fae247, entries=150, sequenceid=14, filesize=11.7 K 2024-12-04T19:14:31,298 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 6fbcb032a86ef6909d8f2378183195f0 in 1109ms, sequenceid=14, compaction requested=false 2024-12-04T19:14:31,298 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6fbcb032a86ef6909d8f2378183195f0: 2024-12-04T19:14:31,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-04T19:14:31,416 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-04T19:14:31,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:31,417 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2902): Flushing 6fbcb032a86ef6909d8f2378183195f0 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-12-04T19:14:31,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6fbcb032a86ef6909d8f2378183195f0, store=A 2024-12-04T19:14:31,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T19:14:31,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6fbcb032a86ef6909d8f2378183195f0, store=B 2024-12-04T19:14:31,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T19:14:31,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6fbcb032a86ef6909d8f2378183195f0, store=C 2024-12-04T19:14:31,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T19:14:31,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204df46ed2f552546ada170041ede8c9633_6fbcb032a86ef6909d8f2378183195f0 is 50, key is test_row_0/A:col10/1733339670322/Put/seqid=0 2024-12-04T19:14:31,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741970_1146 (size=12154) 2024-12-04T19:14:31,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:31,459 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204df46ed2f552546ada170041ede8c9633_6fbcb032a86ef6909d8f2378183195f0 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204df46ed2f552546ada170041ede8c9633_6fbcb032a86ef6909d8f2378183195f0 2024-12-04T19:14:31,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/A/ca0f1cf71d0643ab941daaa49659a425, store: [table=TestAcidGuarantees family=A region=6fbcb032a86ef6909d8f2378183195f0] 2024-12-04T19:14:31,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/A/ca0f1cf71d0643ab941daaa49659a425 is 175, key is test_row_0/A:col10/1733339670322/Put/seqid=0 2024-12-04T19:14:31,475 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. as already flushing 2024-12-04T19:14:31,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(8855): Flush requested on 6fbcb032a86ef6909d8f2378183195f0 2024-12-04T19:14:31,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741971_1147 (size=30955) 2024-12-04T19:14:31,496 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=49, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/A/ca0f1cf71d0643ab941daaa49659a425 2024-12-04T19:14:31,499 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:31,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52102 deadline: 1733339731487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:31,500 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:31,500 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:31,500 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:31,500 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:31,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52122 deadline: 1733339731489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:31,501 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:31,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52066 deadline: 1733339731489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:31,501 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:31,502 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:31,502 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:31,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52088 deadline: 1733339731491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:31,502 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:31,502 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:31,502 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:31,502 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:31,502 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:31,502 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:31,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52124 deadline: 1733339731491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:31,502 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:31,502 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:31,503 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:31,503 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:31,503 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:31,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/B/551081890a48437a94cb83422ae43980 is 50, key is test_row_0/B:col10/1733339670322/Put/seqid=0 2024-12-04T19:14:31,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741972_1148 (size=12001) 2024-12-04T19:14:31,551 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/B/551081890a48437a94cb83422ae43980 2024-12-04T19:14:31,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/C/a4ae14e5e2714d06aa76e3baec83b50b is 50, key is test_row_0/C:col10/1733339670322/Put/seqid=0 2024-12-04T19:14:31,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741973_1149 (size=12001) 2024-12-04T19:14:31,587 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/C/a4ae14e5e2714d06aa76e3baec83b50b 2024-12-04T19:14:31,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/A/ca0f1cf71d0643ab941daaa49659a425 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/ca0f1cf71d0643ab941daaa49659a425 2024-12-04T19:14:31,619 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/ca0f1cf71d0643ab941daaa49659a425, entries=150, sequenceid=49, filesize=30.2 K 2024-12-04T19:14:31,620 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:31,620 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:31,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52124 deadline: 1733339731613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:31,621 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:31,621 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:31,621 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:31,621 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:31,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52088 deadline: 1733339731613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:31,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/B/551081890a48437a94cb83422ae43980 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/551081890a48437a94cb83422ae43980 2024-12-04T19:14:31,626 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:31,626 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:31,626 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:31,626 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:31,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52122 deadline: 1733339731615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:31,627 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:31,627 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:31,627 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:31,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52066 deadline: 1733339731615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:31,627 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:31,627 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:31,627 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:31,627 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:31,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52102 deadline: 1733339731612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:31,628 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:31,628 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:31,628 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:31,640 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/551081890a48437a94cb83422ae43980, entries=150, sequenceid=49, filesize=11.7 K 2024-12-04T19:14:31,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/C/a4ae14e5e2714d06aa76e3baec83b50b as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/a4ae14e5e2714d06aa76e3baec83b50b 2024-12-04T19:14:31,655 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/a4ae14e5e2714d06aa76e3baec83b50b, entries=150, sequenceid=49, filesize=11.7 K 2024-12-04T19:14:31,658 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=53.67 KB/54960 for 6fbcb032a86ef6909d8f2378183195f0 in 241ms, sequenceid=49, compaction requested=false 2024-12-04T19:14:31,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2603): Flush status journal for 6fbcb032a86ef6909d8f2378183195f0: 2024-12-04T19:14:31,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:31,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=39 2024-12-04T19:14:31,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4169): Remote procedure done, pid=39 2024-12-04T19:14:31,666 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-12-04T19:14:31,666 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4860 sec 2024-12-04T19:14:31,670 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees in 1.4970 sec 2024-12-04T19:14:31,838 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6fbcb032a86ef6909d8f2378183195f0 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-04T19:14:31,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6fbcb032a86ef6909d8f2378183195f0, store=A 2024-12-04T19:14:31,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:14:31,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6fbcb032a86ef6909d8f2378183195f0, store=B 2024-12-04T19:14:31,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:14:31,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6fbcb032a86ef6909d8f2378183195f0, store=C 2024-12-04T19:14:31,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:14:31,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(8855): Flush requested on 6fbcb032a86ef6909d8f2378183195f0 2024-12-04T19:14:31,858 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412040b13366b55b2412b9fbcf388cfee31bf_6fbcb032a86ef6909d8f2378183195f0 is 50, key is test_row_0/A:col10/1733339671836/Put/seqid=0 2024-12-04T19:14:31,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741974_1150 (size=12154) 2024-12-04T19:14:31,883 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733339671863, max timestamp=1733339671877Num uniques 859; 2024-12-04T19:14:31,883 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733339671863, max timestamp=1733339671877Num uniques 859; 2024-12-04T19:14:31,883 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T19:14:31,883 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T19:14:31,881 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:31,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52124 deadline: 1733339731875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:31,884 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733339671863, max timestamp=1733339671877Num uniques 859; 2024-12-04T19:14:31,884 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T19:14:31,885 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:31,885 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:31,885 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:31,886 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:31,894 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412040b13366b55b2412b9fbcf388cfee31bf_6fbcb032a86ef6909d8f2378183195f0 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412040b13366b55b2412b9fbcf388cfee31bf_6fbcb032a86ef6909d8f2378183195f0 2024-12-04T19:14:31,895 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/A/24bde5c8899a4fe2abe82b477a05cb48, store: [table=TestAcidGuarantees family=A region=6fbcb032a86ef6909d8f2378183195f0] 2024-12-04T19:14:31,896 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/A/24bde5c8899a4fe2abe82b477a05cb48 is 175, key is test_row_0/A:col10/1733339671836/Put/seqid=0 2024-12-04T19:14:31,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741975_1151 (size=30955) 2024-12-04T19:14:31,920 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:31,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52122 deadline: 1733339731917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:31,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:31,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:31,921 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:31,924 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:31,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52088 deadline: 1733339731919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:31,925 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:31,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52066 deadline: 1733339731920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:31,925 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:31,925 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:31,925 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:31,925 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:31,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52102 deadline: 1733339731922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:31,926 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:31,926 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:31,926 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:31,926 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:31,926 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:31,926 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:31,940 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-04T19:14:31,998 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:31,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52124 deadline: 1733339731991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:31,999 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:31,999 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:31,999 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:32,036 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:32,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52122 deadline: 1733339732032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:32,036 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:32,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:32,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:32,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52088 deadline: 1733339732032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:32,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:32,037 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:32,037 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:32,037 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:32,037 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:32,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52102 deadline: 1733339732032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:32,037 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:32,037 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:32,037 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:32,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52066 deadline: 1733339732033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:32,037 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:32,037 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:32,037 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:32,037 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:32,206 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:32,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52124 deadline: 1733339732202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:32,207 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:32,207 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:32,207 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:32,243 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:32,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52088 deadline: 1733339732242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:32,244 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:32,244 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:32,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52102 deadline: 1733339732243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:32,244 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:32,244 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:32,244 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:32,244 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:32,244 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:32,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52066 deadline: 1733339732243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:32,245 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:32,245 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:32,245 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:32,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52122 deadline: 1733339732243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:32,245 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:32,245 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:32,245 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:32,245 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:32,245 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:32,307 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=62, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/A/24bde5c8899a4fe2abe82b477a05cb48 2024-12-04T19:14:32,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-04T19:14:32,323 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T19:14:32,323 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T19:14:32,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-12-04T19:14:32,325 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/B/d69d333592314c30aac7279c4748c5c9 is 50, key is test_row_0/B:col10/1733339671836/Put/seqid=0 2024-12-04T19:14:32,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-04T19:14:32,326 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T19:14:32,327 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T19:14:32,327 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T19:14:32,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741976_1152 (size=12001) 2024-12-04T19:14:32,342 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/B/d69d333592314c30aac7279c4748c5c9 2024-12-04T19:14:32,354 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/C/ebd2c8f6fed543c6b29eb7b714e0847e is 50, key is test_row_0/C:col10/1733339671836/Put/seqid=0 2024-12-04T19:14:32,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741977_1153 (size=12001) 2024-12-04T19:14:32,381 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/C/ebd2c8f6fed543c6b29eb7b714e0847e 2024-12-04T19:14:32,388 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/A/24bde5c8899a4fe2abe82b477a05cb48 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/24bde5c8899a4fe2abe82b477a05cb48 2024-12-04T19:14:32,396 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/24bde5c8899a4fe2abe82b477a05cb48, entries=150, sequenceid=62, filesize=30.2 K 2024-12-04T19:14:32,398 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/B/d69d333592314c30aac7279c4748c5c9 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/d69d333592314c30aac7279c4748c5c9 2024-12-04T19:14:32,403 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/d69d333592314c30aac7279c4748c5c9, entries=150, sequenceid=62, filesize=11.7 K 2024-12-04T19:14:32,404 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/C/ebd2c8f6fed543c6b29eb7b714e0847e as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/ebd2c8f6fed543c6b29eb7b714e0847e 2024-12-04T19:14:32,415 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/ebd2c8f6fed543c6b29eb7b714e0847e, entries=150, sequenceid=62, filesize=11.7 K 2024-12-04T19:14:32,416 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for 6fbcb032a86ef6909d8f2378183195f0 in 578ms, sequenceid=62, compaction requested=true 2024-12-04T19:14:32,417 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6fbcb032a86ef6909d8f2378183195f0: 2024-12-04T19:14:32,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6fbcb032a86ef6909d8f2378183195f0:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T19:14:32,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:14:32,417 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T19:14:32,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6fbcb032a86ef6909d8f2378183195f0:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T19:14:32,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T19:14:32,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6fbcb032a86ef6909d8f2378183195f0:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T19:14:32,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-04T19:14:32,417 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T19:14:32,418 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T19:14:32,418 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1541): 6fbcb032a86ef6909d8f2378183195f0/A is initiating minor compaction (all files) 2024-12-04T19:14:32,418 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6fbcb032a86ef6909d8f2378183195f0/A in TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:32,419 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/f1de4ce9a5da4ec5b0f7c61e4d79a706, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/ca0f1cf71d0643ab941daaa49659a425, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/24bde5c8899a4fe2abe82b477a05cb48] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp, totalSize=99.1 K 2024-12-04T19:14:32,419 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:32,419 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. files: [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/f1de4ce9a5da4ec5b0f7c61e4d79a706, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/ca0f1cf71d0643ab941daaa49659a425, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/24bde5c8899a4fe2abe82b477a05cb48] 2024-12-04T19:14:32,420 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T19:14:32,420 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1541): 6fbcb032a86ef6909d8f2378183195f0/B is initiating minor compaction (all files) 2024-12-04T19:14:32,420 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6fbcb032a86ef6909d8f2378183195f0/B in TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:32,420 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/e83d0ddf343c45fca0441609aff38cf5, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/551081890a48437a94cb83422ae43980, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/d69d333592314c30aac7279c4748c5c9] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp, totalSize=35.2 K 2024-12-04T19:14:32,420 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting f1de4ce9a5da4ec5b0f7c61e4d79a706, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733339670183 2024-12-04T19:14:32,420 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting e83d0ddf343c45fca0441609aff38cf5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733339670183 2024-12-04T19:14:32,421 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting ca0f1cf71d0643ab941daaa49659a425, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733339670319 2024-12-04T19:14:32,421 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 551081890a48437a94cb83422ae43980, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733339670319 2024-12-04T19:14:32,422 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting d69d333592314c30aac7279c4748c5c9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733339671489 2024-12-04T19:14:32,422 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 24bde5c8899a4fe2abe82b477a05cb48, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733339671489 2024-12-04T19:14:32,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-04T19:14:32,445 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=6fbcb032a86ef6909d8f2378183195f0] 2024-12-04T19:14:32,449 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6fbcb032a86ef6909d8f2378183195f0#B#compaction#136 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:14:32,450 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/B/8a9139e5337b41a1bbb61369ff7238bb is 50, key is test_row_0/B:col10/1733339671836/Put/seqid=0 2024-12-04T19:14:32,453 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412043855806de5c6474db4046ce23fc9a8b7_6fbcb032a86ef6909d8f2378183195f0 store=[table=TestAcidGuarantees family=A region=6fbcb032a86ef6909d8f2378183195f0] 2024-12-04T19:14:32,459 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412043855806de5c6474db4046ce23fc9a8b7_6fbcb032a86ef6909d8f2378183195f0, store=[table=TestAcidGuarantees family=A region=6fbcb032a86ef6909d8f2378183195f0] 2024-12-04T19:14:32,460 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412043855806de5c6474db4046ce23fc9a8b7_6fbcb032a86ef6909d8f2378183195f0 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=6fbcb032a86ef6909d8f2378183195f0] 2024-12-04T19:14:32,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741978_1154 (size=12104) 2024-12-04T19:14:32,480 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-04T19:14:32,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:32,481 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing 6fbcb032a86ef6909d8f2378183195f0 3/3 column families, dataSize=114.05 KB heapSize=320.53 KB 2024-12-04T19:14:32,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6fbcb032a86ef6909d8f2378183195f0, store=A 2024-12-04T19:14:32,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T19:14:32,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6fbcb032a86ef6909d8f2378183195f0, store=B 2024-12-04T19:14:32,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T19:14:32,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6fbcb032a86ef6909d8f2378183195f0, store=C 2024-12-04T19:14:32,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T19:14:32,496 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/B/8a9139e5337b41a1bbb61369ff7238bb as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/8a9139e5337b41a1bbb61369ff7238bb 2024-12-04T19:14:32,503 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6fbcb032a86ef6909d8f2378183195f0/B of 6fbcb032a86ef6909d8f2378183195f0 into 8a9139e5337b41a1bbb61369ff7238bb(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:14:32,503 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6fbcb032a86ef6909d8f2378183195f0: 2024-12-04T19:14:32,503 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., storeName=6fbcb032a86ef6909d8f2378183195f0/B, priority=13, startTime=1733339672417; duration=0sec 2024-12-04T19:14:32,504 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T19:14:32,504 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6fbcb032a86ef6909d8f2378183195f0:B 2024-12-04T19:14:32,504 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T19:14:32,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741979_1155 (size=4469) 2024-12-04T19:14:32,506 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T19:14:32,506 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1541): 6fbcb032a86ef6909d8f2378183195f0/C is initiating minor compaction (all files) 2024-12-04T19:14:32,507 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6fbcb032a86ef6909d8f2378183195f0/C in TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:32,507 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/e51ba5f5037348e4afce3ad466fae247, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/a4ae14e5e2714d06aa76e3baec83b50b, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/ebd2c8f6fed543c6b29eb7b714e0847e] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp, totalSize=35.2 K 2024-12-04T19:14:32,508 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting e51ba5f5037348e4afce3ad466fae247, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733339670183 2024-12-04T19:14:32,508 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting a4ae14e5e2714d06aa76e3baec83b50b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733339670319 2024-12-04T19:14:32,508 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6fbcb032a86ef6909d8f2378183195f0#A#compaction#135 average throughput is 0.39 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:14:32,509 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/A/cbc6f06c646b4ecbbea1f4cb572b7b3e is 175, key is test_row_0/A:col10/1733339671836/Put/seqid=0 2024-12-04T19:14:32,510 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting ebd2c8f6fed543c6b29eb7b714e0847e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733339671489 2024-12-04T19:14:32,522 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. as already flushing 2024-12-04T19:14:32,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(8855): Flush requested on 6fbcb032a86ef6909d8f2378183195f0 2024-12-04T19:14:32,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741980_1156 (size=31058) 2024-12-04T19:14:32,562 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6fbcb032a86ef6909d8f2378183195f0#C#compaction#138 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:14:32,563 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/C/25355aa88e5c44929671b0407beb3d1b is 50, key is test_row_0/C:col10/1733339671836/Put/seqid=0 2024-12-04T19:14:32,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412045039d25f09a54b9fac07f952a177ae91_6fbcb032a86ef6909d8f2378183195f0 is 50, key is test_row_0/A:col10/1733339671911/Put/seqid=0 2024-12-04T19:14:32,571 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:32,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52088 deadline: 1733339732562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:32,571 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:32,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52122 deadline: 1733339732566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:32,571 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:32,572 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:32,572 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:32,572 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:32,572 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:32,572 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:32,572 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:32,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52102 deadline: 1733339732566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:32,572 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:32,573 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:32,573 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:32,576 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:32,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52066 deadline: 1733339732571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:32,576 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:32,576 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:32,576 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:32,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52124 deadline: 1733339732571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:32,576 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:32,577 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:32,577 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:32,577 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:32,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741981_1157 (size=12104) 2024-12-04T19:14:32,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741982_1158 (size=12154) 2024-12-04T19:14:32,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:32,604 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412045039d25f09a54b9fac07f952a177ae91_6fbcb032a86ef6909d8f2378183195f0 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412045039d25f09a54b9fac07f952a177ae91_6fbcb032a86ef6909d8f2378183195f0 2024-12-04T19:14:32,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/A/b492be17977f491fa34b1a403da3c778, store: [table=TestAcidGuarantees family=A region=6fbcb032a86ef6909d8f2378183195f0] 2024-12-04T19:14:32,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/A/b492be17977f491fa34b1a403da3c778 is 175, key is test_row_0/A:col10/1733339671911/Put/seqid=0 2024-12-04T19:14:32,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741983_1159 (size=30955) 2024-12-04T19:14:32,624 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=95, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/A/b492be17977f491fa34b1a403da3c778 2024-12-04T19:14:32,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-04T19:14:32,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/B/acdaa6f2fa304f5982d055ad6202187f is 50, key is test_row_0/B:col10/1733339671911/Put/seqid=0 2024-12-04T19:14:32,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741984_1160 (size=12001) 2024-12-04T19:14:32,685 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/B/acdaa6f2fa304f5982d055ad6202187f 2024-12-04T19:14:32,696 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:32,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52088 deadline: 1733339732682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:32,696 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:32,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52122 deadline: 1733339732683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:32,697 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:32,697 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:32,697 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:32,697 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:32,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52066 deadline: 1733339732683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:32,698 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:32,698 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:32,698 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:32,698 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:32,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:32,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52102 deadline: 1733339732683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:32,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52124 deadline: 1733339732683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:32,698 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:32,698 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:32,698 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:32,699 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:32,699 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:32,699 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:32,699 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:32,699 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:32,699 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:32,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/C/d3ecfd4e2363449cb0dcc76ec1b4fd2f is 50, key is test_row_0/C:col10/1733339671911/Put/seqid=0 2024-12-04T19:14:32,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741985_1161 (size=12001) 2024-12-04T19:14:32,729 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/C/d3ecfd4e2363449cb0dcc76ec1b4fd2f 2024-12-04T19:14:32,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/A/b492be17977f491fa34b1a403da3c778 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/b492be17977f491fa34b1a403da3c778 2024-12-04T19:14:32,746 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/b492be17977f491fa34b1a403da3c778, entries=150, sequenceid=95, filesize=30.2 K 2024-12-04T19:14:32,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/B/acdaa6f2fa304f5982d055ad6202187f as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/acdaa6f2fa304f5982d055ad6202187f 2024-12-04T19:14:32,756 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/acdaa6f2fa304f5982d055ad6202187f, entries=150, sequenceid=95, filesize=11.7 K 2024-12-04T19:14:32,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/C/d3ecfd4e2363449cb0dcc76ec1b4fd2f as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/d3ecfd4e2363449cb0dcc76ec1b4fd2f 2024-12-04T19:14:32,768 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/d3ecfd4e2363449cb0dcc76ec1b4fd2f, entries=150, sequenceid=95, filesize=11.7 K 2024-12-04T19:14:32,770 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~320.48 KB/328176, currentSize=73.80 KB/75570 for 6fbcb032a86ef6909d8f2378183195f0 in 289ms, sequenceid=95, compaction requested=false 2024-12-04T19:14:32,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for 6fbcb032a86ef6909d8f2378183195f0: 2024-12-04T19:14:32,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:32,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-12-04T19:14:32,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-12-04T19:14:32,776 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-12-04T19:14:32,776 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 444 msec 2024-12-04T19:14:32,778 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 454 msec 2024-12-04T19:14:32,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(8855): Flush requested on 6fbcb032a86ef6909d8f2378183195f0 2024-12-04T19:14:32,904 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6fbcb032a86ef6909d8f2378183195f0 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-04T19:14:32,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6fbcb032a86ef6909d8f2378183195f0, store=A 2024-12-04T19:14:32,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:14:32,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6fbcb032a86ef6909d8f2378183195f0, store=B 2024-12-04T19:14:32,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:14:32,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6fbcb032a86ef6909d8f2378183195f0, store=C 2024-12-04T19:14:32,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:14:32,912 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204aa02a948c3b547af9d2c3cbc3b2fa163_6fbcb032a86ef6909d8f2378183195f0 is 50, key is test_row_0/A:col10/1733339672561/Put/seqid=0 2024-12-04T19:14:32,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741986_1162 (size=14594) 2024-12-04T19:14:32,948 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/A/cbc6f06c646b4ecbbea1f4cb572b7b3e as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/cbc6f06c646b4ecbbea1f4cb572b7b3e 2024-12-04T19:14:32,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-04T19:14:32,952 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T19:14:32,954 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T19:14:32,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-12-04T19:14:32,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-04T19:14:32,958 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6fbcb032a86ef6909d8f2378183195f0/A of 6fbcb032a86ef6909d8f2378183195f0 into cbc6f06c646b4ecbbea1f4cb572b7b3e(size=30.3 K), total size for store is 60.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:14:32,958 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6fbcb032a86ef6909d8f2378183195f0: 2024-12-04T19:14:32,958 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., storeName=6fbcb032a86ef6909d8f2378183195f0/A, priority=13, startTime=1733339672417; duration=0sec 2024-12-04T19:14:32,959 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:14:32,959 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6fbcb032a86ef6909d8f2378183195f0:A 2024-12-04T19:14:32,959 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T19:14:32,962 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T19:14:32,962 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T19:14:32,973 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:32,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52122 deadline: 1733339732959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:32,973 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:32,973 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:32,974 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:32,974 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:32,974 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:32,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52088 deadline: 1733339732959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:32,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52102 deadline: 1733339732961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:32,974 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:32,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52124 deadline: 1733339732963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:32,974 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:32,974 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:32,974 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:32,975 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:32,975 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:32,975 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:32,978 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:32,978 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:32,978 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:32,979 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:32,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52066 deadline: 1733339732972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:32,979 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:32,979 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:32,979 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:33,003 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/C/25355aa88e5c44929671b0407beb3d1b as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/25355aa88e5c44929671b0407beb3d1b 2024-12-04T19:14:33,011 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6fbcb032a86ef6909d8f2378183195f0/C of 6fbcb032a86ef6909d8f2378183195f0 into 25355aa88e5c44929671b0407beb3d1b(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:14:33,011 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6fbcb032a86ef6909d8f2378183195f0: 2024-12-04T19:14:33,011 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., storeName=6fbcb032a86ef6909d8f2378183195f0/C, priority=13, startTime=1733339672417; duration=0sec 2024-12-04T19:14:33,011 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:14:33,011 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6fbcb032a86ef6909d8f2378183195f0:C 2024-12-04T19:14:33,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-04T19:14:33,084 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:33,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52122 deadline: 1733339733082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:33,085 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:33,085 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:33,085 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:33,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52088 deadline: 1733339733082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:33,085 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:33,085 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:33,085 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:33,085 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:33,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52066 deadline: 1733339733083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:33,086 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:33,086 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:33,086 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:33,086 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:33,086 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:33,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52102 deadline: 1733339733083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:33,086 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:33,086 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:33,086 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:33,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52124 deadline: 1733339733083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:33,086 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:33,087 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:33,087 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:33,087 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:33,115 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-04T19:14:33,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:33,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. as already flushing 2024-12-04T19:14:33,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:33,115 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:33,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:33,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:33,269 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-04T19:14:33,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:33,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. as already flushing 2024-12-04T19:14:33,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:33,269 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:33,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:33,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:33,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-04T19:14:33,292 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:33,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52088 deadline: 1733339733292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:33,292 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:33,293 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:33,293 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:33,293 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:33,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52122 deadline: 1733339733292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:33,293 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:33,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52066 deadline: 1733339733292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:33,293 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:33,293 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:33,293 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:33,293 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:33,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52124 deadline: 1733339733292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:33,293 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:33,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52102 deadline: 1733339733292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:33,294 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:33,294 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:33,294 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:33,294 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:33,294 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:33,294 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:33,294 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:33,294 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:33,294 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:33,324 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:33,329 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204aa02a948c3b547af9d2c3cbc3b2fa163_6fbcb032a86ef6909d8f2378183195f0 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204aa02a948c3b547af9d2c3cbc3b2fa163_6fbcb032a86ef6909d8f2378183195f0 2024-12-04T19:14:33,330 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/A/8bcb1e32dab940fab7509fb7597bbfdd, store: [table=TestAcidGuarantees family=A region=6fbcb032a86ef6909d8f2378183195f0] 2024-12-04T19:14:33,331 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/A/8bcb1e32dab940fab7509fb7597bbfdd is 175, key is test_row_0/A:col10/1733339672561/Put/seqid=0 2024-12-04T19:14:33,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741987_1163 (size=39549) 2024-12-04T19:14:33,422 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-04T19:14:33,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:33,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. as already flushing 2024-12-04T19:14:33,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:33,423 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:33,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:33,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:33,576 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-04T19:14:33,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:33,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. as already flushing 2024-12-04T19:14:33,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:33,576 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:33,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:33,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:33,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-04T19:14:33,605 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:33,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52088 deadline: 1733339733602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:33,606 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:33,606 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:33,606 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:33,606 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:33,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52122 deadline: 1733339733602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:33,606 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:33,606 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:33,606 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:33,606 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:33,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52066 deadline: 1733339733602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:33,607 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:33,607 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:33,607 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:33,607 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:33,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52124 deadline: 1733339733602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:33,607 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:33,607 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:33,607 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:33,607 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:33,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52102 deadline: 1733339733602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:33,608 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:33,608 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:33,608 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:33,729 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-04T19:14:33,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:33,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. as already flushing 2024-12-04T19:14:33,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:33,729 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:33,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:33,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:33,737 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=111, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/A/8bcb1e32dab940fab7509fb7597bbfdd 2024-12-04T19:14:33,745 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/B/38f2c7f850f345789efcde7eab8062fd is 50, key is test_row_0/B:col10/1733339672561/Put/seqid=0 2024-12-04T19:14:33,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741988_1164 (size=12001) 2024-12-04T19:14:33,758 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=111 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/B/38f2c7f850f345789efcde7eab8062fd 2024-12-04T19:14:33,770 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/C/d39e2d243fdc460aba6c6ce2e8682e11 is 50, key is test_row_0/C:col10/1733339672561/Put/seqid=0 2024-12-04T19:14:33,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741989_1165 (size=12001) 2024-12-04T19:14:33,779 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=111 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/C/d39e2d243fdc460aba6c6ce2e8682e11 2024-12-04T19:14:33,800 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/A/8bcb1e32dab940fab7509fb7597bbfdd as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/8bcb1e32dab940fab7509fb7597bbfdd 2024-12-04T19:14:33,805 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/8bcb1e32dab940fab7509fb7597bbfdd, entries=200, sequenceid=111, filesize=38.6 K 2024-12-04T19:14:33,811 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/B/38f2c7f850f345789efcde7eab8062fd as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/38f2c7f850f345789efcde7eab8062fd 2024-12-04T19:14:33,817 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/38f2c7f850f345789efcde7eab8062fd, entries=150, sequenceid=111, filesize=11.7 K 2024-12-04T19:14:33,821 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/C/d39e2d243fdc460aba6c6ce2e8682e11 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/d39e2d243fdc460aba6c6ce2e8682e11 2024-12-04T19:14:33,826 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/d39e2d243fdc460aba6c6ce2e8682e11, entries=150, sequenceid=111, filesize=11.7 K 2024-12-04T19:14:33,832 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 6fbcb032a86ef6909d8f2378183195f0 in 929ms, sequenceid=111, compaction requested=true 2024-12-04T19:14:33,832 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6fbcb032a86ef6909d8f2378183195f0: 2024-12-04T19:14:33,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6fbcb032a86ef6909d8f2378183195f0:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T19:14:33,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:14:33,832 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T19:14:33,833 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T19:14:33,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6fbcb032a86ef6909d8f2378183195f0:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T19:14:33,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:14:33,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6fbcb032a86ef6909d8f2378183195f0:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T19:14:33,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T19:14:33,834 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T19:14:33,834 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1541): 6fbcb032a86ef6909d8f2378183195f0/B is initiating minor compaction (all files) 2024-12-04T19:14:33,834 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6fbcb032a86ef6909d8f2378183195f0/B in TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:33,835 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/8a9139e5337b41a1bbb61369ff7238bb, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/acdaa6f2fa304f5982d055ad6202187f, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/38f2c7f850f345789efcde7eab8062fd] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp, totalSize=35.3 K 2024-12-04T19:14:33,835 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T19:14:33,835 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1541): 6fbcb032a86ef6909d8f2378183195f0/A is initiating minor compaction (all files) 2024-12-04T19:14:33,835 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6fbcb032a86ef6909d8f2378183195f0/A in TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:33,835 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/cbc6f06c646b4ecbbea1f4cb572b7b3e, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/b492be17977f491fa34b1a403da3c778, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/8bcb1e32dab940fab7509fb7597bbfdd] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp, totalSize=99.2 K 2024-12-04T19:14:33,835 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:33,835 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. files: [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/cbc6f06c646b4ecbbea1f4cb572b7b3e, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/b492be17977f491fa34b1a403da3c778, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/8bcb1e32dab940fab7509fb7597bbfdd] 2024-12-04T19:14:33,836 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting cbc6f06c646b4ecbbea1f4cb572b7b3e, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733339671489 2024-12-04T19:14:33,836 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a9139e5337b41a1bbb61369ff7238bb, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733339671489 2024-12-04T19:14:33,836 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting acdaa6f2fa304f5982d055ad6202187f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733339671911 2024-12-04T19:14:33,836 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting b492be17977f491fa34b1a403da3c778, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733339671911 2024-12-04T19:14:33,837 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 38f2c7f850f345789efcde7eab8062fd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=111, earliestPutTs=1733339672561 2024-12-04T19:14:33,837 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8bcb1e32dab940fab7509fb7597bbfdd, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=111, earliestPutTs=1733339672561 2024-12-04T19:14:33,853 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=6fbcb032a86ef6909d8f2378183195f0] 2024-12-04T19:14:33,854 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6fbcb032a86ef6909d8f2378183195f0#B#compaction#144 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:14:33,855 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/B/d47a7f08b5674ae9acbdd428e6689c5c is 50, key is test_row_0/B:col10/1733339672561/Put/seqid=0 2024-12-04T19:14:33,861 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241204ea59109956bc403d9f4471b8f86224d1_6fbcb032a86ef6909d8f2378183195f0 store=[table=TestAcidGuarantees family=A region=6fbcb032a86ef6909d8f2378183195f0] 2024-12-04T19:14:33,864 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241204ea59109956bc403d9f4471b8f86224d1_6fbcb032a86ef6909d8f2378183195f0, store=[table=TestAcidGuarantees family=A region=6fbcb032a86ef6909d8f2378183195f0] 2024-12-04T19:14:33,864 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204ea59109956bc403d9f4471b8f86224d1_6fbcb032a86ef6909d8f2378183195f0 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=6fbcb032a86ef6909d8f2378183195f0] 2024-12-04T19:14:33,882 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-04T19:14:33,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:33,882 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing 6fbcb032a86ef6909d8f2378183195f0 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-04T19:14:33,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6fbcb032a86ef6909d8f2378183195f0, store=A 2024-12-04T19:14:33,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:14:33,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6fbcb032a86ef6909d8f2378183195f0, store=B 2024-12-04T19:14:33,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:14:33,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6fbcb032a86ef6909d8f2378183195f0, store=C 2024-12-04T19:14:33,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:14:33,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741990_1166 (size=12207) 2024-12-04T19:14:33,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741991_1167 (size=4469) 2024-12-04T19:14:33,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204000df985678347b3bf85c43ffa4b4327_6fbcb032a86ef6909d8f2378183195f0 is 50, key is test_row_0/A:col10/1733339672970/Put/seqid=0 2024-12-04T19:14:33,905 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6fbcb032a86ef6909d8f2378183195f0#A#compaction#145 average throughput is 0.48 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:14:33,905 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/A/081d3d75cfcf4440b0874f81a6c5910b is 175, key is test_row_0/A:col10/1733339672561/Put/seqid=0 2024-12-04T19:14:33,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741993_1169 (size=31161) 2024-12-04T19:14:33,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741992_1168 (size=12154) 2024-12-04T19:14:33,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:33,927 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204000df985678347b3bf85c43ffa4b4327_6fbcb032a86ef6909d8f2378183195f0 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204000df985678347b3bf85c43ffa4b4327_6fbcb032a86ef6909d8f2378183195f0 2024-12-04T19:14:33,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/A/43ab5e2e9ac94c468544b00e4dcb19f1, store: [table=TestAcidGuarantees family=A region=6fbcb032a86ef6909d8f2378183195f0] 2024-12-04T19:14:33,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/A/43ab5e2e9ac94c468544b00e4dcb19f1 is 175, key is test_row_0/A:col10/1733339672970/Put/seqid=0 2024-12-04T19:14:33,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741994_1170 (size=30955) 2024-12-04T19:14:33,941 INFO [master/9ba2734cd39f:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-12-04T19:14:33,941 INFO [master/9ba2734cd39f:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-12-04T19:14:34,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-04T19:14:34,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(8855): Flush requested on 6fbcb032a86ef6909d8f2378183195f0 2024-12-04T19:14:34,113 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. as already flushing 2024-12-04T19:14:34,131 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:34,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52088 deadline: 1733339734126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:34,131 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:34,131 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:34,131 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:34,135 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:34,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52102 deadline: 1733339734128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:34,135 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:34,135 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:34,135 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:34,135 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:34,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52122 deadline: 1733339734130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:34,136 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:34,136 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:34,136 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:34,138 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:34,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52066 deadline: 1733339734131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:34,138 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:34,139 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:34,139 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:34,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52124 deadline: 1733339734131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:34,139 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:34,139 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:34,139 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:34,139 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:34,242 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:34,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52088 deadline: 1733339734241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:34,242 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:34,242 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:34,242 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:34,244 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:34,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52102 deadline: 1733339734242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:34,244 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:34,244 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:34,245 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:34,245 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:34,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52122 deadline: 1733339734242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:34,245 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:34,245 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:34,245 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:34,245 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:34,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52066 deadline: 1733339734242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:34,245 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:34,245 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:34,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52124 deadline: 1733339734242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:34,245 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:34,245 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:34,245 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:34,246 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:34,246 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:34,300 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/B/d47a7f08b5674ae9acbdd428e6689c5c as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/d47a7f08b5674ae9acbdd428e6689c5c 2024-12-04T19:14:34,305 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6fbcb032a86ef6909d8f2378183195f0/B of 6fbcb032a86ef6909d8f2378183195f0 into d47a7f08b5674ae9acbdd428e6689c5c(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:14:34,305 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6fbcb032a86ef6909d8f2378183195f0: 2024-12-04T19:14:34,305 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., storeName=6fbcb032a86ef6909d8f2378183195f0/B, priority=13, startTime=1733339673832; duration=0sec 2024-12-04T19:14:34,305 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T19:14:34,306 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6fbcb032a86ef6909d8f2378183195f0:B 2024-12-04T19:14:34,306 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T19:14:34,307 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T19:14:34,307 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1541): 6fbcb032a86ef6909d8f2378183195f0/C is initiating minor compaction (all files) 2024-12-04T19:14:34,307 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6fbcb032a86ef6909d8f2378183195f0/C in TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:34,307 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/25355aa88e5c44929671b0407beb3d1b, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/d3ecfd4e2363449cb0dcc76ec1b4fd2f, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/d39e2d243fdc460aba6c6ce2e8682e11] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp, totalSize=35.3 K 2024-12-04T19:14:34,307 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 25355aa88e5c44929671b0407beb3d1b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733339671489 2024-12-04T19:14:34,307 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting d3ecfd4e2363449cb0dcc76ec1b4fd2f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733339671911 2024-12-04T19:14:34,308 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting d39e2d243fdc460aba6c6ce2e8682e11, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=111, earliestPutTs=1733339672561 2024-12-04T19:14:34,317 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6fbcb032a86ef6909d8f2378183195f0#C#compaction#147 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:14:34,317 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/C/127107817f34420d91cba0daae31db6c is 50, key is test_row_0/C:col10/1733339672561/Put/seqid=0 2024-12-04T19:14:34,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741995_1171 (size=12207) 2024-12-04T19:14:34,327 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/A/081d3d75cfcf4440b0874f81a6c5910b as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/081d3d75cfcf4440b0874f81a6c5910b 2024-12-04T19:14:34,332 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6fbcb032a86ef6909d8f2378183195f0/A of 6fbcb032a86ef6909d8f2378183195f0 into 081d3d75cfcf4440b0874f81a6c5910b(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:14:34,332 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6fbcb032a86ef6909d8f2378183195f0: 2024-12-04T19:14:34,332 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., storeName=6fbcb032a86ef6909d8f2378183195f0/A, priority=13, startTime=1733339673832; duration=0sec 2024-12-04T19:14:34,332 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:14:34,333 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6fbcb032a86ef6909d8f2378183195f0:A 2024-12-04T19:14:34,337 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=130, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/A/43ab5e2e9ac94c468544b00e4dcb19f1 2024-12-04T19:14:34,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/B/621e27b2f2824f13940958479d95705c is 50, key is test_row_0/B:col10/1733339672970/Put/seqid=0 2024-12-04T19:14:34,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741996_1172 (size=12001) 2024-12-04T19:14:34,386 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/B/621e27b2f2824f13940958479d95705c 2024-12-04T19:14:34,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/C/74b40f3fff454313afa5b77040c52225 is 50, key is test_row_0/C:col10/1733339672970/Put/seqid=0 2024-12-04T19:14:34,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741997_1173 (size=12001) 2024-12-04T19:14:34,417 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/C/74b40f3fff454313afa5b77040c52225 2024-12-04T19:14:34,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/A/43ab5e2e9ac94c468544b00e4dcb19f1 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/43ab5e2e9ac94c468544b00e4dcb19f1 2024-12-04T19:14:34,433 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/43ab5e2e9ac94c468544b00e4dcb19f1, entries=150, sequenceid=130, filesize=30.2 K 2024-12-04T19:14:34,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/B/621e27b2f2824f13940958479d95705c as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/621e27b2f2824f13940958479d95705c 2024-12-04T19:14:34,442 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/621e27b2f2824f13940958479d95705c, entries=150, sequenceid=130, filesize=11.7 K 2024-12-04T19:14:34,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/C/74b40f3fff454313afa5b77040c52225 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/74b40f3fff454313afa5b77040c52225 2024-12-04T19:14:34,451 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/74b40f3fff454313afa5b77040c52225, entries=150, sequenceid=130, filesize=11.7 K 2024-12-04T19:14:34,452 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=87.22 KB/89310 for 6fbcb032a86ef6909d8f2378183195f0 in 570ms, sequenceid=130, compaction requested=false 2024-12-04T19:14:34,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for 6fbcb032a86ef6909d8f2378183195f0: 2024-12-04T19:14:34,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:34,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-12-04T19:14:34,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-12-04T19:14:34,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(8855): Flush requested on 6fbcb032a86ef6909d8f2378183195f0 2024-12-04T19:14:34,455 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6fbcb032a86ef6909d8f2378183195f0 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-04T19:14:34,457 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-12-04T19:14:34,457 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4920 sec 2024-12-04T19:14:34,459 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 1.5030 sec 2024-12-04T19:14:34,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6fbcb032a86ef6909d8f2378183195f0, store=A 2024-12-04T19:14:34,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:14:34,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6fbcb032a86ef6909d8f2378183195f0, store=B 2024-12-04T19:14:34,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:14:34,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6fbcb032a86ef6909d8f2378183195f0, store=C 2024-12-04T19:14:34,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:14:34,470 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204ddc662beb8224a169953fec38deaddba_6fbcb032a86ef6909d8f2378183195f0 is 50, key is test_row_0/A:col10/1733339674123/Put/seqid=0 2024-12-04T19:14:34,481 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:34,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52124 deadline: 1733339734475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:34,482 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:34,482 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:34,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52088 deadline: 1733339734475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:34,482 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:34,482 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:34,482 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:34,482 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:34,482 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:34,482 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:34,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52122 deadline: 1733339734477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:34,483 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:34,483 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:34,483 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:34,486 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:34,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52066 deadline: 1733339734480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:34,486 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:34,486 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:34,486 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:34,488 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:34,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52102 deadline: 1733339734482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:34,490 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:34,490 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:34,490 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:34,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741998_1174 (size=12304) 2024-12-04T19:14:34,496 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:34,502 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204ddc662beb8224a169953fec38deaddba_6fbcb032a86ef6909d8f2378183195f0 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204ddc662beb8224a169953fec38deaddba_6fbcb032a86ef6909d8f2378183195f0 2024-12-04T19:14:34,503 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/A/4acbdc9afbf648e393c8596d01b3bcb5, store: [table=TestAcidGuarantees family=A region=6fbcb032a86ef6909d8f2378183195f0] 2024-12-04T19:14:34,504 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/A/4acbdc9afbf648e393c8596d01b3bcb5 is 175, key is test_row_0/A:col10/1733339674123/Put/seqid=0 2024-12-04T19:14:34,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741999_1175 (size=31105) 2024-12-04T19:14:34,595 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:34,595 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:34,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52088 deadline: 1733339734591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:34,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52124 deadline: 1733339734591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:34,595 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:34,595 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:34,596 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:34,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52122 deadline: 1733339734592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:34,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52102 deadline: 1733339734592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:34,596 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:34,596 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:34,596 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:34,596 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:34,596 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:34,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52066 deadline: 1733339734592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:34,596 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:34,596 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:34,596 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:34,596 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:34,596 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:34,596 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:34,596 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:34,596 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:34,596 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:34,596 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:34,728 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/C/127107817f34420d91cba0daae31db6c as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/127107817f34420d91cba0daae31db6c 2024-12-04T19:14:34,734 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6fbcb032a86ef6909d8f2378183195f0/C of 6fbcb032a86ef6909d8f2378183195f0 into 127107817f34420d91cba0daae31db6c(size=11.9 K), total size for store is 23.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:14:34,734 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6fbcb032a86ef6909d8f2378183195f0: 2024-12-04T19:14:34,734 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., storeName=6fbcb032a86ef6909d8f2378183195f0/C, priority=13, startTime=1733339673833; duration=0sec 2024-12-04T19:14:34,735 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:14:34,735 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6fbcb032a86ef6909d8f2378183195f0:C 2024-12-04T19:14:34,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:34,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:34,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52124 deadline: 1733339734801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:34,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52122 deadline: 1733339734802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:34,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:34,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52102 deadline: 1733339734802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:34,807 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:34,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:34,807 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:34,807 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:34,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52066 deadline: 1733339734802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:34,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:34,808 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:34,808 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:34,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52088 deadline: 1733339734802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:34,808 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:34,808 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:34,808 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:34,808 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:34,808 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:34,808 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:34,808 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:34,808 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:34,808 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:34,808 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:34,928 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=150, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/A/4acbdc9afbf648e393c8596d01b3bcb5 2024-12-04T19:14:34,936 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/B/bbce90ea7b044c13856aa17fda134dd8 is 50, key is test_row_0/B:col10/1733339674123/Put/seqid=0 2024-12-04T19:14:34,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742000_1176 (size=12151) 2024-12-04T19:14:34,941 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=150 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/B/bbce90ea7b044c13856aa17fda134dd8 2024-12-04T19:14:34,947 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/C/64e269733fe541ebb69d47cb22fbacc4 is 50, key is test_row_0/C:col10/1733339674123/Put/seqid=0 2024-12-04T19:14:34,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742001_1177 (size=12151) 2024-12-04T19:14:34,951 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=150 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/C/64e269733fe541ebb69d47cb22fbacc4 2024-12-04T19:14:34,957 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/A/4acbdc9afbf648e393c8596d01b3bcb5 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/4acbdc9afbf648e393c8596d01b3bcb5 2024-12-04T19:14:34,961 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/4acbdc9afbf648e393c8596d01b3bcb5, entries=150, sequenceid=150, filesize=30.4 K 2024-12-04T19:14:34,962 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/B/bbce90ea7b044c13856aa17fda134dd8 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/bbce90ea7b044c13856aa17fda134dd8 2024-12-04T19:14:34,967 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/bbce90ea7b044c13856aa17fda134dd8, entries=150, sequenceid=150, filesize=11.9 K 2024-12-04T19:14:34,968 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/C/64e269733fe541ebb69d47cb22fbacc4 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/64e269733fe541ebb69d47cb22fbacc4 2024-12-04T19:14:34,973 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/64e269733fe541ebb69d47cb22fbacc4, entries=150, sequenceid=150, filesize=11.9 K 2024-12-04T19:14:34,973 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 6fbcb032a86ef6909d8f2378183195f0 in 518ms, sequenceid=150, compaction requested=true 2024-12-04T19:14:34,974 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6fbcb032a86ef6909d8f2378183195f0: 2024-12-04T19:14:34,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6fbcb032a86ef6909d8f2378183195f0:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T19:14:34,974 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T19:14:34,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:14:34,974 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T19:14:34,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6fbcb032a86ef6909d8f2378183195f0:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T19:14:34,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:14:34,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6fbcb032a86ef6909d8f2378183195f0:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T19:14:34,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T19:14:34,975 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93221 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T19:14:34,975 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T19:14:34,975 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1541): 6fbcb032a86ef6909d8f2378183195f0/B is initiating minor compaction (all files) 2024-12-04T19:14:34,975 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1541): 6fbcb032a86ef6909d8f2378183195f0/A is initiating minor compaction (all files) 2024-12-04T19:14:34,975 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6fbcb032a86ef6909d8f2378183195f0/B in TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:34,975 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6fbcb032a86ef6909d8f2378183195f0/A in TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:34,975 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/d47a7f08b5674ae9acbdd428e6689c5c, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/621e27b2f2824f13940958479d95705c, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/bbce90ea7b044c13856aa17fda134dd8] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp, totalSize=35.5 K 2024-12-04T19:14:34,975 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/081d3d75cfcf4440b0874f81a6c5910b, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/43ab5e2e9ac94c468544b00e4dcb19f1, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/4acbdc9afbf648e393c8596d01b3bcb5] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp, totalSize=91.0 K 2024-12-04T19:14:34,975 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:34,975 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. files: [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/081d3d75cfcf4440b0874f81a6c5910b, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/43ab5e2e9ac94c468544b00e4dcb19f1, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/4acbdc9afbf648e393c8596d01b3bcb5] 2024-12-04T19:14:34,976 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting d47a7f08b5674ae9acbdd428e6689c5c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=111, earliestPutTs=1733339672561 2024-12-04T19:14:34,976 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 081d3d75cfcf4440b0874f81a6c5910b, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=111, earliestPutTs=1733339672561 2024-12-04T19:14:34,976 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 621e27b2f2824f13940958479d95705c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1733339672912 2024-12-04T19:14:34,976 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 43ab5e2e9ac94c468544b00e4dcb19f1, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1733339672912 2024-12-04T19:14:34,976 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting bbce90ea7b044c13856aa17fda134dd8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1733339674123 2024-12-04T19:14:34,977 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4acbdc9afbf648e393c8596d01b3bcb5, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1733339674123 2024-12-04T19:14:34,983 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=6fbcb032a86ef6909d8f2378183195f0] 2024-12-04T19:14:34,984 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6fbcb032a86ef6909d8f2378183195f0#B#compaction#153 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:14:34,985 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/B/891deabd92bb4d6c9d9d318abb75557f is 50, key is test_row_0/B:col10/1733339674123/Put/seqid=0 2024-12-04T19:14:34,985 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241204446ce7bd5fc0417a836bb262f1045235_6fbcb032a86ef6909d8f2378183195f0 store=[table=TestAcidGuarantees family=A region=6fbcb032a86ef6909d8f2378183195f0] 2024-12-04T19:14:34,987 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241204446ce7bd5fc0417a836bb262f1045235_6fbcb032a86ef6909d8f2378183195f0, store=[table=TestAcidGuarantees family=A region=6fbcb032a86ef6909d8f2378183195f0] 2024-12-04T19:14:34,988 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204446ce7bd5fc0417a836bb262f1045235_6fbcb032a86ef6909d8f2378183195f0 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=6fbcb032a86ef6909d8f2378183195f0] 2024-12-04T19:14:34,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742002_1178 (size=12459) 2024-12-04T19:14:35,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742003_1179 (size=4469) 2024-12-04T19:14:35,006 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6fbcb032a86ef6909d8f2378183195f0#A#compaction#154 average throughput is 1.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:14:35,007 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/A/4a51e4252b7a43eeb6f38c925c362222 is 175, key is test_row_0/A:col10/1733339674123/Put/seqid=0 2024-12-04T19:14:35,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742004_1180 (size=31413) 2024-12-04T19:14:35,016 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/A/4a51e4252b7a43eeb6f38c925c362222 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/4a51e4252b7a43eeb6f38c925c362222 2024-12-04T19:14:35,021 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6fbcb032a86ef6909d8f2378183195f0/A of 6fbcb032a86ef6909d8f2378183195f0 into 4a51e4252b7a43eeb6f38c925c362222(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:14:35,021 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6fbcb032a86ef6909d8f2378183195f0: 2024-12-04T19:14:35,021 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., storeName=6fbcb032a86ef6909d8f2378183195f0/A, priority=13, startTime=1733339674974; duration=0sec 2024-12-04T19:14:35,021 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T19:14:35,021 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6fbcb032a86ef6909d8f2378183195f0:A 2024-12-04T19:14:35,021 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T19:14:35,022 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T19:14:35,022 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1541): 6fbcb032a86ef6909d8f2378183195f0/C is initiating minor compaction (all files) 2024-12-04T19:14:35,022 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6fbcb032a86ef6909d8f2378183195f0/C in TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:35,023 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/127107817f34420d91cba0daae31db6c, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/74b40f3fff454313afa5b77040c52225, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/64e269733fe541ebb69d47cb22fbacc4] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp, totalSize=35.5 K 2024-12-04T19:14:35,023 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 127107817f34420d91cba0daae31db6c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=111, earliestPutTs=1733339672561 2024-12-04T19:14:35,023 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 74b40f3fff454313afa5b77040c52225, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1733339672912 2024-12-04T19:14:35,024 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 64e269733fe541ebb69d47cb22fbacc4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1733339674123 2024-12-04T19:14:35,032 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6fbcb032a86ef6909d8f2378183195f0#C#compaction#155 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:14:35,033 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/C/38e52066eb70483d95af7ee12f286fa3 is 50, key is test_row_0/C:col10/1733339674123/Put/seqid=0 2024-12-04T19:14:35,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742005_1181 (size=12459) 2024-12-04T19:14:35,045 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/C/38e52066eb70483d95af7ee12f286fa3 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/38e52066eb70483d95af7ee12f286fa3 2024-12-04T19:14:35,050 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6fbcb032a86ef6909d8f2378183195f0/C of 6fbcb032a86ef6909d8f2378183195f0 into 38e52066eb70483d95af7ee12f286fa3(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:14:35,050 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6fbcb032a86ef6909d8f2378183195f0: 2024-12-04T19:14:35,050 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., storeName=6fbcb032a86ef6909d8f2378183195f0/C, priority=13, startTime=1733339674974; duration=0sec 2024-12-04T19:14:35,050 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:14:35,050 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6fbcb032a86ef6909d8f2378183195f0:C 2024-12-04T19:14:35,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-04T19:14:35,102 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T19:14:35,103 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T19:14:35,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-12-04T19:14:35,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-04T19:14:35,104 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T19:14:35,105 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T19:14:35,105 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T19:14:35,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(8855): Flush requested on 6fbcb032a86ef6909d8f2378183195f0 2024-12-04T19:14:35,113 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6fbcb032a86ef6909d8f2378183195f0 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-04T19:14:35,113 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6fbcb032a86ef6909d8f2378183195f0, store=A 2024-12-04T19:14:35,113 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:14:35,113 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6fbcb032a86ef6909d8f2378183195f0, store=B 2024-12-04T19:14:35,113 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:14:35,113 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6fbcb032a86ef6909d8f2378183195f0, store=C 2024-12-04T19:14:35,113 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:14:35,119 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120449fb09bdcddc45afab85361cd7c3b5a8_6fbcb032a86ef6909d8f2378183195f0 is 50, key is test_row_0/A:col10/1733339674478/Put/seqid=0 2024-12-04T19:14:35,136 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:35,136 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:35,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52122 deadline: 1733339735127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:35,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52102 deadline: 1733339735126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:35,136 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:35,136 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:35,136 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:35,136 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:35,136 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:35,136 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:35,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52066 deadline: 1733339735134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:35,136 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:35,137 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:35,137 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:35,137 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:35,140 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:35,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52088 deadline: 1733339735136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:35,141 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:35,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52124 deadline: 1733339735136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:35,141 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:35,141 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:35,141 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:35,141 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:35,141 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:35,141 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:35,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742006_1182 (size=14794) 2024-12-04T19:14:35,145 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:35,149 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120449fb09bdcddc45afab85361cd7c3b5a8_6fbcb032a86ef6909d8f2378183195f0 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120449fb09bdcddc45afab85361cd7c3b5a8_6fbcb032a86ef6909d8f2378183195f0 2024-12-04T19:14:35,151 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/A/f4f82e6b7972423e9b604e329e5f2b18, store: [table=TestAcidGuarantees family=A region=6fbcb032a86ef6909d8f2378183195f0] 2024-12-04T19:14:35,151 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/A/f4f82e6b7972423e9b604e329e5f2b18 is 175, key is test_row_0/A:col10/1733339674478/Put/seqid=0 2024-12-04T19:14:35,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742007_1183 (size=39749) 2024-12-04T19:14:35,159 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=168, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/A/f4f82e6b7972423e9b604e329e5f2b18 2024-12-04T19:14:35,173 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/B/9ac9c5e06a1f40ff8f93ea724e2cdaf4 is 50, key is test_row_0/B:col10/1733339674478/Put/seqid=0 2024-12-04T19:14:35,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742008_1184 (size=12151) 2024-12-04T19:14:35,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-04T19:14:35,242 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:35,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52102 deadline: 1733339735241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:35,243 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:35,243 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:35,243 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:35,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52122 deadline: 1733339735241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:35,243 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:35,243 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:35,243 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:35,243 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:35,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52066 deadline: 1733339735241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:35,243 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:35,243 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:35,243 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:35,243 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:35,252 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:35,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52124 deadline: 1733339735252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:35,253 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:35,253 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:35,253 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:35,253 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:35,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52088 deadline: 1733339735252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:35,253 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:35,253 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:35,253 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:35,257 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-04T19:14:35,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:35,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. as already flushing 2024-12-04T19:14:35,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:35,258 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:35,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:35,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:35,402 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/B/891deabd92bb4d6c9d9d318abb75557f as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/891deabd92bb4d6c9d9d318abb75557f 2024-12-04T19:14:35,407 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6fbcb032a86ef6909d8f2378183195f0/B of 6fbcb032a86ef6909d8f2378183195f0 into 891deabd92bb4d6c9d9d318abb75557f(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:14:35,407 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6fbcb032a86ef6909d8f2378183195f0: 2024-12-04T19:14:35,407 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., storeName=6fbcb032a86ef6909d8f2378183195f0/B, priority=13, startTime=1733339674974; duration=0sec 2024-12-04T19:14:35,407 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:14:35,407 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6fbcb032a86ef6909d8f2378183195f0:B 2024-12-04T19:14:35,409 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-04T19:14:35,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:35,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. as already flushing 2024-12-04T19:14:35,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:35,410 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:35,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:35,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:35,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-04T19:14:35,452 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:35,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52102 deadline: 1733339735451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:35,452 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:35,452 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:35,452 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:35,453 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:35,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52066 deadline: 1733339735451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:35,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52122 deadline: 1733339735451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:35,453 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:35,453 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:35,453 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:35,453 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:35,453 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:35,453 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:35,453 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:35,463 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:35,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52124 deadline: 1733339735462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:35,463 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:35,463 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:35,463 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:35,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52088 deadline: 1733339735462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:35,463 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:35,463 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:35,463 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:35,463 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:35,562 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-04T19:14:35,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:35,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. as already flushing 2024-12-04T19:14:35,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:35,563 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:35,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:35,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:35,577 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/B/9ac9c5e06a1f40ff8f93ea724e2cdaf4 2024-12-04T19:14:35,585 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/C/588ef0f1f7fa4759b9ed7ea9b5f52fcf is 50, key is test_row_0/C:col10/1733339674478/Put/seqid=0 2024-12-04T19:14:35,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742009_1185 (size=12151) 2024-12-04T19:14:35,715 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-04T19:14:35,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:35,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. as already flushing 2024-12-04T19:14:35,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:35,715 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:35,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:35,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:35,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-04T19:14:35,765 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:35,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52102 deadline: 1733339735761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:35,766 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:35,766 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:35,766 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:35,766 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:35,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52122 deadline: 1733339735761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:35,766 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:35,766 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:35,766 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:35,766 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:35,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52066 deadline: 1733339735761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:35,767 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:35,767 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:35,767 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:35,775 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:35,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52124 deadline: 1733339735772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:35,775 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:35,775 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:35,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52088 deadline: 1733339735772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:35,775 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:35,775 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:35,775 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:35,775 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:35,775 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:35,868 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-04T19:14:35,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:35,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. as already flushing 2024-12-04T19:14:35,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:35,868 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:35,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:35,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:35,989 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/C/588ef0f1f7fa4759b9ed7ea9b5f52fcf 2024-12-04T19:14:35,993 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/A/f4f82e6b7972423e9b604e329e5f2b18 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/f4f82e6b7972423e9b604e329e5f2b18 2024-12-04T19:14:35,996 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/f4f82e6b7972423e9b604e329e5f2b18, entries=200, sequenceid=168, filesize=38.8 K 2024-12-04T19:14:35,998 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/B/9ac9c5e06a1f40ff8f93ea724e2cdaf4 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/9ac9c5e06a1f40ff8f93ea724e2cdaf4 2024-12-04T19:14:36,002 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/9ac9c5e06a1f40ff8f93ea724e2cdaf4, entries=150, sequenceid=168, filesize=11.9 K 2024-12-04T19:14:36,003 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/C/588ef0f1f7fa4759b9ed7ea9b5f52fcf as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/588ef0f1f7fa4759b9ed7ea9b5f52fcf 2024-12-04T19:14:36,007 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/588ef0f1f7fa4759b9ed7ea9b5f52fcf, entries=150, sequenceid=168, filesize=11.9 K 2024-12-04T19:14:36,008 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for 6fbcb032a86ef6909d8f2378183195f0 in 895ms, sequenceid=168, compaction requested=false 2024-12-04T19:14:36,008 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6fbcb032a86ef6909d8f2378183195f0: 2024-12-04T19:14:36,021 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-04T19:14:36,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:36,021 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing 6fbcb032a86ef6909d8f2378183195f0 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-04T19:14:36,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6fbcb032a86ef6909d8f2378183195f0, store=A 2024-12-04T19:14:36,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:14:36,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6fbcb032a86ef6909d8f2378183195f0, store=B 2024-12-04T19:14:36,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:14:36,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6fbcb032a86ef6909d8f2378183195f0, store=C 2024-12-04T19:14:36,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:14:36,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412044a6dd891a98543a0a6861ae6f18e6e9e_6fbcb032a86ef6909d8f2378183195f0 is 50, key is test_row_0/A:col10/1733339675131/Put/seqid=0 2024-12-04T19:14:36,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742010_1186 (size=12304) 2024-12-04T19:14:36,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-04T19:14:36,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(8855): Flush requested on 6fbcb032a86ef6909d8f2378183195f0 2024-12-04T19:14:36,276 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. as already flushing 2024-12-04T19:14:36,288 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:36,288 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:36,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52066 deadline: 1733339736285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:36,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52124 deadline: 1733339736286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:36,289 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:36,289 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:36,289 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:36,289 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:36,289 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:36,289 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:36,293 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:36,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52102 deadline: 1733339736287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:36,293 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:36,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52122 deadline: 1733339736288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:36,294 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:36,294 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:36,294 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:36,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52088 deadline: 1733339736289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:36,294 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:36,294 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:36,294 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:36,294 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:36,294 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:36,294 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:36,294 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:36,394 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:36,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52124 deadline: 1733339736392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:36,394 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:36,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52066 deadline: 1733339736392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:36,394 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:36,394 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:36,395 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:36,395 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:36,395 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:36,395 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:36,407 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:36,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52102 deadline: 1733339736402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:36,407 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:36,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52122 deadline: 1733339736402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:36,407 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:36,407 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:36,407 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:36,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52088 deadline: 1733339736402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:36,407 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:36,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:36,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:36,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:36,408 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:36,408 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:36,408 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:36,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:36,437 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412044a6dd891a98543a0a6861ae6f18e6e9e_6fbcb032a86ef6909d8f2378183195f0 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412044a6dd891a98543a0a6861ae6f18e6e9e_6fbcb032a86ef6909d8f2378183195f0 2024-12-04T19:14:36,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/A/c008e7e326054e5c9a9f05d574a17a58, store: [table=TestAcidGuarantees family=A region=6fbcb032a86ef6909d8f2378183195f0] 2024-12-04T19:14:36,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/A/c008e7e326054e5c9a9f05d574a17a58 is 175, key is test_row_0/A:col10/1733339675131/Put/seqid=0 2024-12-04T19:14:36,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742011_1187 (size=31105) 2024-12-04T19:14:36,603 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:36,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52124 deadline: 1733339736602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:36,604 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:36,604 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:36,604 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:36,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52066 deadline: 1733339736602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:36,604 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:36,604 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:36,604 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:36,604 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:36,613 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:36,613 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:36,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52122 deadline: 1733339736611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:36,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52102 deadline: 1733339736611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:36,613 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:36,613 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:36,613 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:36,613 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:36,613 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:36,613 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:36,613 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:36,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52088 deadline: 1733339736611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:36,614 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:36,614 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:36,614 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:36,843 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=187, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/A/c008e7e326054e5c9a9f05d574a17a58 2024-12-04T19:14:36,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/B/610fef7ddfea402891bb5cd70796136b is 50, key is test_row_0/B:col10/1733339675131/Put/seqid=0 2024-12-04T19:14:36,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742012_1188 (size=12151) 2024-12-04T19:14:36,914 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:36,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52124 deadline: 1733339736912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:36,915 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:36,915 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:36,915 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:36,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:36,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52066 deadline: 1733339736912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:36,915 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:36,915 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:36,915 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:36,924 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:36,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52122 deadline: 1733339736922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:36,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:36,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:36,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:36,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:36,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52102 deadline: 1733339736922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:36,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:36,926 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:36,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52088 deadline: 1733339736922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:36,926 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:36,926 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:36,926 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:36,926 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:36,926 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:37,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-04T19:14:37,256 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=187 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/B/610fef7ddfea402891bb5cd70796136b 2024-12-04T19:14:37,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/C/ead88345670e4b1aa1cf2a091a656399 is 50, key is test_row_0/C:col10/1733339675131/Put/seqid=0 2024-12-04T19:14:37,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742013_1189 (size=12151) 2024-12-04T19:14:37,425 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:37,425 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:37,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52066 deadline: 1733339737422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:37,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52124 deadline: 1733339737422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:37,425 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:37,425 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:37,425 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:37,426 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:37,426 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:37,426 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:37,434 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:37,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52122 deadline: 1733339737431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:37,435 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:37,435 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:37,435 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:37,435 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:37,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52102 deadline: 1733339737431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:37,435 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:37,435 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:37,435 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:37,435 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:37,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52088 deadline: 1733339737432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:37,436 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:37,436 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:37,436 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:37,667 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=187 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/C/ead88345670e4b1aa1cf2a091a656399 2024-12-04T19:14:37,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/A/c008e7e326054e5c9a9f05d574a17a58 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/c008e7e326054e5c9a9f05d574a17a58 2024-12-04T19:14:37,756 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/c008e7e326054e5c9a9f05d574a17a58, entries=150, sequenceid=187, filesize=30.4 K 2024-12-04T19:14:37,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/B/610fef7ddfea402891bb5cd70796136b as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/610fef7ddfea402891bb5cd70796136b 2024-12-04T19:14:37,762 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/610fef7ddfea402891bb5cd70796136b, entries=150, sequenceid=187, filesize=11.9 K 2024-12-04T19:14:37,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/C/ead88345670e4b1aa1cf2a091a656399 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/ead88345670e4b1aa1cf2a091a656399 2024-12-04T19:14:37,767 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/ead88345670e4b1aa1cf2a091a656399, entries=150, sequenceid=187, filesize=11.9 K 2024-12-04T19:14:37,769 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 6fbcb032a86ef6909d8f2378183195f0 in 1747ms, sequenceid=187, compaction requested=true 2024-12-04T19:14:37,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 6fbcb032a86ef6909d8f2378183195f0: 2024-12-04T19:14:37,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:37,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-12-04T19:14:37,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-12-04T19:14:37,773 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-12-04T19:14:37,773 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6660 sec 2024-12-04T19:14:37,774 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 2.6700 sec 2024-12-04T19:14:38,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(8855): Flush requested on 6fbcb032a86ef6909d8f2378183195f0 2024-12-04T19:14:38,445 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6fbcb032a86ef6909d8f2378183195f0 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-04T19:14:38,446 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6fbcb032a86ef6909d8f2378183195f0, store=A 2024-12-04T19:14:38,446 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:14:38,446 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6fbcb032a86ef6909d8f2378183195f0, store=B 2024-12-04T19:14:38,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:14:38,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6fbcb032a86ef6909d8f2378183195f0, store=C 2024-12-04T19:14:38,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:14:38,455 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204140f0642e2d2442aab37e0b87fb4cc1e_6fbcb032a86ef6909d8f2378183195f0 is 50, key is test_row_0/A:col10/1733339678445/Put/seqid=0 2024-12-04T19:14:38,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742014_1190 (size=17284) 2024-12-04T19:14:38,462 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:38,467 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204140f0642e2d2442aab37e0b87fb4cc1e_6fbcb032a86ef6909d8f2378183195f0 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204140f0642e2d2442aab37e0b87fb4cc1e_6fbcb032a86ef6909d8f2378183195f0 2024-12-04T19:14:38,467 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:38,467 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:38,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52124 deadline: 1733339738461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:38,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52122 deadline: 1733339738463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:38,468 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:38,468 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:38,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52066 deadline: 1733339738464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:38,468 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:38,468 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:38,468 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:38,468 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:38,468 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:38,468 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:38,468 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:38,468 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:38,469 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/A/ddf441de1c58454093001a36ada4665a, store: [table=TestAcidGuarantees family=A region=6fbcb032a86ef6909d8f2378183195f0] 2024-12-04T19:14:38,470 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/A/ddf441de1c58454093001a36ada4665a is 175, key is test_row_0/A:col10/1733339678445/Put/seqid=0 2024-12-04T19:14:38,473 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:38,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52102 deadline: 1733339738468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:38,473 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:38,473 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:38,473 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:38,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52088 deadline: 1733339738468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:38,473 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:38,474 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:38,474 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:38,474 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:38,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742015_1191 (size=48389) 2024-12-04T19:14:38,575 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:38,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52124 deadline: 1733339738571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:38,575 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:38,576 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:38,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52122 deadline: 1733339738571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:38,576 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:38,576 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:38,576 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:38,576 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:38,576 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:38,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52066 deadline: 1733339738572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:38,576 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:38,576 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:38,576 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:38,576 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:38,583 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:38,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52102 deadline: 1733339738582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:38,584 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:38,584 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:38,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52088 deadline: 1733339738582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:38,584 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:38,584 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:38,584 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:38,584 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:38,584 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:38,785 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:38,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52124 deadline: 1733339738782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:38,785 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:38,785 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:38,786 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:38,786 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:38,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52122 deadline: 1733339738782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:38,786 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:38,786 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:38,786 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:38,786 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:38,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52066 deadline: 1733339738782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:38,786 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:38,786 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:38,786 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:38,795 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:38,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52102 deadline: 1733339738791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:38,795 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:38,795 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:38,795 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:38,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52088 deadline: 1733339738791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:38,796 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:38,796 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:38,796 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:38,796 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:38,878 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=203, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/A/ddf441de1c58454093001a36ada4665a 2024-12-04T19:14:38,886 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/B/aa7ee4925ba64e3ebcab1681c4b5451f is 50, key is test_row_0/B:col10/1733339678445/Put/seqid=0 2024-12-04T19:14:38,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742016_1192 (size=12151) 2024-12-04T19:14:38,898 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/B/aa7ee4925ba64e3ebcab1681c4b5451f 2024-12-04T19:14:38,911 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/C/1f6865ca0c9d4c3b8574577aab7af30e is 50, key is test_row_0/C:col10/1733339678445/Put/seqid=0 2024-12-04T19:14:38,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742017_1193 (size=12151) 2024-12-04T19:14:38,924 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/C/1f6865ca0c9d4c3b8574577aab7af30e 2024-12-04T19:14:38,930 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/A/ddf441de1c58454093001a36ada4665a as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/ddf441de1c58454093001a36ada4665a 2024-12-04T19:14:38,938 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/ddf441de1c58454093001a36ada4665a, entries=250, sequenceid=203, filesize=47.3 K 2024-12-04T19:14:38,939 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/B/aa7ee4925ba64e3ebcab1681c4b5451f as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/aa7ee4925ba64e3ebcab1681c4b5451f 2024-12-04T19:14:38,944 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/aa7ee4925ba64e3ebcab1681c4b5451f, entries=150, sequenceid=203, filesize=11.9 K 2024-12-04T19:14:38,945 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/C/1f6865ca0c9d4c3b8574577aab7af30e as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/1f6865ca0c9d4c3b8574577aab7af30e 2024-12-04T19:14:38,952 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/1f6865ca0c9d4c3b8574577aab7af30e, entries=150, sequenceid=203, filesize=11.9 K 2024-12-04T19:14:38,953 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 6fbcb032a86ef6909d8f2378183195f0 in 508ms, sequenceid=203, compaction requested=true 2024-12-04T19:14:38,953 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6fbcb032a86ef6909d8f2378183195f0: 2024-12-04T19:14:38,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6fbcb032a86ef6909d8f2378183195f0:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T19:14:38,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:14:38,953 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T19:14:38,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6fbcb032a86ef6909d8f2378183195f0:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T19:14:38,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:14:38,953 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T19:14:38,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6fbcb032a86ef6909d8f2378183195f0:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T19:14:38,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T19:14:38,955 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 150656 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T19:14:38,955 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1541): 6fbcb032a86ef6909d8f2378183195f0/A is initiating minor compaction (all files) 2024-12-04T19:14:38,955 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6fbcb032a86ef6909d8f2378183195f0/A in TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:38,955 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/4a51e4252b7a43eeb6f38c925c362222, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/f4f82e6b7972423e9b604e329e5f2b18, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/c008e7e326054e5c9a9f05d574a17a58, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/ddf441de1c58454093001a36ada4665a] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp, totalSize=147.1 K 2024-12-04T19:14:38,955 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:38,955 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. files: [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/4a51e4252b7a43eeb6f38c925c362222, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/f4f82e6b7972423e9b604e329e5f2b18, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/c008e7e326054e5c9a9f05d574a17a58, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/ddf441de1c58454093001a36ada4665a] 2024-12-04T19:14:38,955 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48912 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T19:14:38,956 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1541): 6fbcb032a86ef6909d8f2378183195f0/B is initiating minor compaction (all files) 2024-12-04T19:14:38,956 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6fbcb032a86ef6909d8f2378183195f0/B in TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:38,956 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/891deabd92bb4d6c9d9d318abb75557f, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/9ac9c5e06a1f40ff8f93ea724e2cdaf4, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/610fef7ddfea402891bb5cd70796136b, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/aa7ee4925ba64e3ebcab1681c4b5451f] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp, totalSize=47.8 K 2024-12-04T19:14:38,956 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4a51e4252b7a43eeb6f38c925c362222, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1733339674123 2024-12-04T19:14:38,956 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 891deabd92bb4d6c9d9d318abb75557f, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1733339674123 2024-12-04T19:14:38,956 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting f4f82e6b7972423e9b604e329e5f2b18, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1733339674472 2024-12-04T19:14:38,956 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ac9c5e06a1f40ff8f93ea724e2cdaf4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1733339674472 2024-12-04T19:14:38,957 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting c008e7e326054e5c9a9f05d574a17a58, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1733339675122 2024-12-04T19:14:38,957 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 610fef7ddfea402891bb5cd70796136b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1733339675122 2024-12-04T19:14:38,957 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting ddf441de1c58454093001a36ada4665a, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1733339676283 2024-12-04T19:14:38,958 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting aa7ee4925ba64e3ebcab1681c4b5451f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1733339676286 2024-12-04T19:14:38,966 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=6fbcb032a86ef6909d8f2378183195f0] 2024-12-04T19:14:38,984 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6fbcb032a86ef6909d8f2378183195f0#B#compaction#166 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:14:38,984 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120462a89ee2ab15495bbf5aa691b2019248_6fbcb032a86ef6909d8f2378183195f0 store=[table=TestAcidGuarantees family=A region=6fbcb032a86ef6909d8f2378183195f0] 2024-12-04T19:14:38,984 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/B/7eec59502ecd4134a1fd60367199d946 is 50, key is test_row_0/B:col10/1733339678445/Put/seqid=0 2024-12-04T19:14:38,986 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120462a89ee2ab15495bbf5aa691b2019248_6fbcb032a86ef6909d8f2378183195f0, store=[table=TestAcidGuarantees family=A region=6fbcb032a86ef6909d8f2378183195f0] 2024-12-04T19:14:38,987 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120462a89ee2ab15495bbf5aa691b2019248_6fbcb032a86ef6909d8f2378183195f0 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=6fbcb032a86ef6909d8f2378183195f0] 2024-12-04T19:14:39,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742018_1194 (size=12595) 2024-12-04T19:14:39,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742019_1195 (size=4469) 2024-12-04T19:14:39,012 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6fbcb032a86ef6909d8f2378183195f0#A#compaction#165 average throughput is 0.53 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:14:39,012 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/A/06ab8ea5ec5a4d308fb491f4e2dc3fcb is 175, key is test_row_0/A:col10/1733339678445/Put/seqid=0 2024-12-04T19:14:39,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742020_1196 (size=31549) 2024-12-04T19:14:39,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(8855): Flush requested on 6fbcb032a86ef6909d8f2378183195f0 2024-12-04T19:14:39,096 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6fbcb032a86ef6909d8f2378183195f0 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-04T19:14:39,096 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6fbcb032a86ef6909d8f2378183195f0, store=A 2024-12-04T19:14:39,096 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:14:39,096 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6fbcb032a86ef6909d8f2378183195f0, store=B 2024-12-04T19:14:39,096 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:14:39,096 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6fbcb032a86ef6909d8f2378183195f0, store=C 2024-12-04T19:14:39,096 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:14:39,104 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204989cd70cec92495289017730a671d0b7_6fbcb032a86ef6909d8f2378183195f0 is 50, key is test_row_0/A:col10/1733339679094/Put/seqid=0 2024-12-04T19:14:39,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742021_1197 (size=14794) 2024-12-04T19:14:39,130 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:39,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52122 deadline: 1733339739123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:39,130 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:39,130 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:39,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52124 deadline: 1733339739123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:39,130 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:39,131 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:39,131 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:39,131 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:39,131 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:39,132 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:39,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52102 deadline: 1733339739129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:39,132 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:39,132 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:39,132 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:39,135 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:39,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52066 deadline: 1733339739130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:39,136 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:39,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52088 deadline: 1733339739130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:39,136 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:39,136 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:39,136 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:39,137 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:39,137 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:39,137 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:39,243 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:39,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52122 deadline: 1733339739241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:39,243 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:39,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:39,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:39,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52124 deadline: 1733339739241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:39,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:39,244 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:39,244 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:39,244 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:39,244 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:39,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52102 deadline: 1733339739241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:39,244 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:39,244 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:39,244 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:39,244 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:39,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52088 deadline: 1733339739242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:39,244 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:39,245 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:39,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52066 deadline: 1733339739242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:39,245 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:39,245 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:39,245 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:39,245 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:39,245 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:39,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-04T19:14:39,271 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T19:14:39,272 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T19:14:39,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-12-04T19:14:39,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-04T19:14:39,274 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T19:14:39,274 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T19:14:39,274 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T19:14:39,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-04T19:14:39,407 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/B/7eec59502ecd4134a1fd60367199d946 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/7eec59502ecd4134a1fd60367199d946 2024-12-04T19:14:39,413 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6fbcb032a86ef6909d8f2378183195f0/B of 6fbcb032a86ef6909d8f2378183195f0 into 7eec59502ecd4134a1fd60367199d946(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:14:39,413 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6fbcb032a86ef6909d8f2378183195f0: 2024-12-04T19:14:39,413 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., storeName=6fbcb032a86ef6909d8f2378183195f0/B, priority=12, startTime=1733339678953; duration=0sec 2024-12-04T19:14:39,413 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T19:14:39,413 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6fbcb032a86ef6909d8f2378183195f0:B 2024-12-04T19:14:39,413 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T19:14:39,415 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48912 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T19:14:39,415 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1541): 6fbcb032a86ef6909d8f2378183195f0/C is initiating minor compaction (all files) 2024-12-04T19:14:39,415 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6fbcb032a86ef6909d8f2378183195f0/C in TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:39,415 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/38e52066eb70483d95af7ee12f286fa3, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/588ef0f1f7fa4759b9ed7ea9b5f52fcf, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/ead88345670e4b1aa1cf2a091a656399, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/1f6865ca0c9d4c3b8574577aab7af30e] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp, totalSize=47.8 K 2024-12-04T19:14:39,415 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 38e52066eb70483d95af7ee12f286fa3, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1733339674123 2024-12-04T19:14:39,416 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 588ef0f1f7fa4759b9ed7ea9b5f52fcf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1733339674472 2024-12-04T19:14:39,416 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting ead88345670e4b1aa1cf2a091a656399, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1733339675122 2024-12-04T19:14:39,416 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f6865ca0c9d4c3b8574577aab7af30e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1733339676286 2024-12-04T19:14:39,422 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/A/06ab8ea5ec5a4d308fb491f4e2dc3fcb as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/06ab8ea5ec5a4d308fb491f4e2dc3fcb 2024-12-04T19:14:39,424 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6fbcb032a86ef6909d8f2378183195f0#C#compaction#168 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:14:39,425 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/C/23421e154e064cf6860ce386f6c86dc2 is 50, key is test_row_0/C:col10/1733339678445/Put/seqid=0 2024-12-04T19:14:39,426 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-04T19:14:39,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:39,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. as already flushing 2024-12-04T19:14:39,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:39,427 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:39,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:39,428 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6fbcb032a86ef6909d8f2378183195f0/A of 6fbcb032a86ef6909d8f2378183195f0 into 06ab8ea5ec5a4d308fb491f4e2dc3fcb(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:14:39,428 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6fbcb032a86ef6909d8f2378183195f0: 2024-12-04T19:14:39,428 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., storeName=6fbcb032a86ef6909d8f2378183195f0/A, priority=12, startTime=1733339678953; duration=0sec 2024-12-04T19:14:39,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:39,428 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:14:39,428 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6fbcb032a86ef6909d8f2378183195f0:A 2024-12-04T19:14:39,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742022_1198 (size=12595) 2024-12-04T19:14:39,455 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:39,455 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:39,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52122 deadline: 1733339739452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:39,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52124 deadline: 1733339739452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:39,455 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:39,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:39,455 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:39,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:39,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52102 deadline: 1733339739452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:39,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:39,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52088 deadline: 1733339739452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:39,456 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:39,456 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:39,456 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:39,456 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:39,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52066 deadline: 1733339739452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:39,456 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:39,456 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:39,456 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:39,456 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:39,456 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:39,456 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:39,456 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:39,456 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:39,456 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:39,523 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:39,527 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204989cd70cec92495289017730a671d0b7_6fbcb032a86ef6909d8f2378183195f0 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204989cd70cec92495289017730a671d0b7_6fbcb032a86ef6909d8f2378183195f0 2024-12-04T19:14:39,528 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/A/947a0beb7118462e81e5ebc19837dd11, store: [table=TestAcidGuarantees family=A region=6fbcb032a86ef6909d8f2378183195f0] 2024-12-04T19:14:39,528 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/A/947a0beb7118462e81e5ebc19837dd11 is 175, key is test_row_0/A:col10/1733339679094/Put/seqid=0 2024-12-04T19:14:39,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742023_1199 (size=39749) 2024-12-04T19:14:39,580 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-04T19:14:39,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:39,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. as already flushing 2024-12-04T19:14:39,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:39,580 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:39,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:39,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:39,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-04T19:14:39,733 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-04T19:14:39,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:39,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. as already flushing 2024-12-04T19:14:39,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:39,733 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:39,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:39,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:39,762 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:39,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52122 deadline: 1733339739762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:39,763 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:39,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52124 deadline: 1733339739762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:39,763 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:39,763 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:39,763 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:39,763 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:39,763 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:39,763 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:39,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52088 deadline: 1733339739762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:39,763 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:39,763 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:39,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52066 deadline: 1733339739762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:39,763 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:39,763 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:39,763 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:39,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52102 deadline: 1733339739762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:39,764 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:39,764 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:39,764 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:39,764 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:39,764 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:39,764 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:39,764 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:39,838 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/C/23421e154e064cf6860ce386f6c86dc2 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/23421e154e064cf6860ce386f6c86dc2 2024-12-04T19:14:39,843 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6fbcb032a86ef6909d8f2378183195f0/C of 6fbcb032a86ef6909d8f2378183195f0 into 23421e154e064cf6860ce386f6c86dc2(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:14:39,843 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6fbcb032a86ef6909d8f2378183195f0: 2024-12-04T19:14:39,843 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., storeName=6fbcb032a86ef6909d8f2378183195f0/C, priority=12, startTime=1733339678953; duration=0sec 2024-12-04T19:14:39,843 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:14:39,843 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6fbcb032a86ef6909d8f2378183195f0:C 2024-12-04T19:14:39,886 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-04T19:14:39,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:39,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. as already flushing 2024-12-04T19:14:39,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:39,886 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:39,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:39,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:39,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-04T19:14:39,933 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=220, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/A/947a0beb7118462e81e5ebc19837dd11 2024-12-04T19:14:39,941 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/B/4f2429a4e66e4ae782dbfaa92de9eabd is 50, key is test_row_0/B:col10/1733339679094/Put/seqid=0 2024-12-04T19:14:39,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742024_1200 (size=12151) 2024-12-04T19:14:39,946 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/B/4f2429a4e66e4ae782dbfaa92de9eabd 2024-12-04T19:14:39,953 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/C/d2911bdede90402c9b9f3f019ec01f33 is 50, key is test_row_0/C:col10/1733339679094/Put/seqid=0 2024-12-04T19:14:39,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742025_1201 (size=12151) 2024-12-04T19:14:40,038 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-04T19:14:40,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:40,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. as already flushing 2024-12-04T19:14:40,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:40,039 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:40,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:40,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:40,191 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-04T19:14:40,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:40,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. as already flushing 2024-12-04T19:14:40,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:40,191 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:40,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:40,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:40,274 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:40,274 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:40,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52122 deadline: 1733339740272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:40,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52124 deadline: 1733339740272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:40,275 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:40,275 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:40,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52102 deadline: 1733339740272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:40,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52088 deadline: 1733339740272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:40,275 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:40,275 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:40,275 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:40,275 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:40,275 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:40,275 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:40,275 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:40,275 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:40,275 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:40,275 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:40,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52066 deadline: 1733339740272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:40,275 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:40,275 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:40,275 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:40,275 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:40,275 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:40,276 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:40,343 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-04T19:14:40,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:40,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. as already flushing 2024-12-04T19:14:40,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:40,344 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:40,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:40,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:40,357 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/C/d2911bdede90402c9b9f3f019ec01f33 2024-12-04T19:14:40,362 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/A/947a0beb7118462e81e5ebc19837dd11 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/947a0beb7118462e81e5ebc19837dd11 2024-12-04T19:14:40,368 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/947a0beb7118462e81e5ebc19837dd11, entries=200, sequenceid=220, filesize=38.8 K 2024-12-04T19:14:40,370 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/B/4f2429a4e66e4ae782dbfaa92de9eabd as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/4f2429a4e66e4ae782dbfaa92de9eabd 2024-12-04T19:14:40,374 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/4f2429a4e66e4ae782dbfaa92de9eabd, entries=150, sequenceid=220, filesize=11.9 K 2024-12-04T19:14:40,375 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/C/d2911bdede90402c9b9f3f019ec01f33 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/d2911bdede90402c9b9f3f019ec01f33 2024-12-04T19:14:40,379 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/d2911bdede90402c9b9f3f019ec01f33, entries=150, sequenceid=220, filesize=11.9 K 2024-12-04T19:14:40,381 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 6fbcb032a86ef6909d8f2378183195f0 in 1285ms, sequenceid=220, compaction requested=false 2024-12-04T19:14:40,381 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6fbcb032a86ef6909d8f2378183195f0: 2024-12-04T19:14:40,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-04T19:14:40,496 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-04T19:14:40,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:40,497 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing 6fbcb032a86ef6909d8f2378183195f0 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-04T19:14:40,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6fbcb032a86ef6909d8f2378183195f0, store=A 2024-12-04T19:14:40,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:14:40,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6fbcb032a86ef6909d8f2378183195f0, store=B 2024-12-04T19:14:40,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:14:40,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6fbcb032a86ef6909d8f2378183195f0, store=C 2024-12-04T19:14:40,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:14:40,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204da4871ac2f0242fbafa194b094af5aaf_6fbcb032a86ef6909d8f2378183195f0 is 50, key is test_row_0/A:col10/1733339679120/Put/seqid=0 2024-12-04T19:14:40,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742026_1202 (size=12304) 2024-12-04T19:14:40,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:40,529 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204da4871ac2f0242fbafa194b094af5aaf_6fbcb032a86ef6909d8f2378183195f0 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204da4871ac2f0242fbafa194b094af5aaf_6fbcb032a86ef6909d8f2378183195f0 2024-12-04T19:14:40,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/A/10ae7d405d6849ef9f2eb062f40303c1, store: [table=TestAcidGuarantees family=A region=6fbcb032a86ef6909d8f2378183195f0] 2024-12-04T19:14:40,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/A/10ae7d405d6849ef9f2eb062f40303c1 is 175, key is test_row_0/A:col10/1733339679120/Put/seqid=0 2024-12-04T19:14:40,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742027_1203 (size=31105) 2024-12-04T19:14:40,938 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=238, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/A/10ae7d405d6849ef9f2eb062f40303c1 2024-12-04T19:14:40,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/B/e16c5b785a4b470d89e7f8f1a69c527b is 50, key is test_row_0/B:col10/1733339679120/Put/seqid=0 2024-12-04T19:14:40,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742028_1204 (size=12151) 2024-12-04T19:14:41,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(8855): Flush requested on 6fbcb032a86ef6909d8f2378183195f0 2024-12-04T19:14:41,283 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. as already flushing 2024-12-04T19:14:41,301 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:41,301 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:41,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52124 deadline: 1733339741297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:41,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52066 deadline: 1733339741298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:41,302 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:41,302 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:41,302 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:41,302 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:41,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52102 deadline: 1733339741299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:41,302 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:41,302 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:41,302 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:41,302 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:41,302 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:41,302 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:41,302 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:41,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52088 deadline: 1733339741300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:41,302 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:41,302 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:41,302 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:41,306 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:41,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52122 deadline: 1733339741301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:41,306 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:41,306 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:41,306 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:41,349 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/B/e16c5b785a4b470d89e7f8f1a69c527b 2024-12-04T19:14:41,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/C/7405d71906e0410b88e76248bb6eaf71 is 50, key is test_row_0/C:col10/1733339679120/Put/seqid=0 2024-12-04T19:14:41,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742029_1205 (size=12151) 2024-12-04T19:14:41,359 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/C/7405d71906e0410b88e76248bb6eaf71 2024-12-04T19:14:41,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/A/10ae7d405d6849ef9f2eb062f40303c1 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/10ae7d405d6849ef9f2eb062f40303c1 2024-12-04T19:14:41,367 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/10ae7d405d6849ef9f2eb062f40303c1, entries=150, sequenceid=238, filesize=30.4 K 2024-12-04T19:14:41,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/B/e16c5b785a4b470d89e7f8f1a69c527b as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/e16c5b785a4b470d89e7f8f1a69c527b 2024-12-04T19:14:41,372 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/e16c5b785a4b470d89e7f8f1a69c527b, entries=150, sequenceid=238, filesize=11.9 K 2024-12-04T19:14:41,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/C/7405d71906e0410b88e76248bb6eaf71 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/7405d71906e0410b88e76248bb6eaf71 2024-12-04T19:14:41,378 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/7405d71906e0410b88e76248bb6eaf71, entries=150, sequenceid=238, filesize=11.9 K 2024-12-04T19:14:41,379 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 6fbcb032a86ef6909d8f2378183195f0 in 881ms, sequenceid=238, compaction requested=true 2024-12-04T19:14:41,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 6fbcb032a86ef6909d8f2378183195f0: 2024-12-04T19:14:41,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:41,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-12-04T19:14:41,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-12-04T19:14:41,382 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-12-04T19:14:41,382 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1060 sec 2024-12-04T19:14:41,383 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 2.1100 sec 2024-12-04T19:14:41,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(8855): Flush requested on 6fbcb032a86ef6909d8f2378183195f0 2024-12-04T19:14:41,415 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6fbcb032a86ef6909d8f2378183195f0 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-04T19:14:41,416 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6fbcb032a86ef6909d8f2378183195f0, store=A 2024-12-04T19:14:41,416 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:14:41,416 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6fbcb032a86ef6909d8f2378183195f0, store=B 2024-12-04T19:14:41,416 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:14:41,416 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6fbcb032a86ef6909d8f2378183195f0, store=C 2024-12-04T19:14:41,416 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:14:41,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-04T19:14:41,422 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T19:14:41,422 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T19:14:41,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-12-04T19:14:41,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-04T19:14:41,424 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T19:14:41,424 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T19:14:41,424 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T19:14:41,426 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204e546894f9b80495c95622f5058babb50_6fbcb032a86ef6909d8f2378183195f0 is 50, key is test_row_0/A:col10/1733339681415/Put/seqid=0 2024-12-04T19:14:41,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742030_1206 (size=17334) 2024-12-04T19:14:41,467 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:41,467 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:41,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52066 deadline: 1733339741459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:41,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52122 deadline: 1733339741460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:41,468 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:41,468 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:41,468 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:41,468 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:41,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52088 deadline: 1733339741460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:41,468 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:41,468 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:41,468 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:41,468 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:41,468 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:41,468 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:41,468 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:41,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52124 deadline: 1733339741466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:41,469 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:41,469 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:41,469 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:41,469 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:41,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52102 deadline: 1733339741467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:41,469 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:41,469 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:41,469 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:41,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-04T19:14:41,576 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-04T19:14:41,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:41,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. as already flushing 2024-12-04T19:14:41,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:41,577 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:41,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:41,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:41,580 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:41,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52122 deadline: 1733339741572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:41,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:41,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:41,580 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:41,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:41,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52066 deadline: 1733339741572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:41,581 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:41,581 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:41,581 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:41,581 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:41,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52088 deadline: 1733339741572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:41,581 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:41,581 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:41,581 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:41,581 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:41,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52124 deadline: 1733339741572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:41,582 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:41,582 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:41,582 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:41,582 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:41,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52102 deadline: 1733339741572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:41,582 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:41,582 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:41,582 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:41,729 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-04T19:14:41,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:41,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. as already flushing 2024-12-04T19:14:41,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:41,729 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:41,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:41,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:41,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-04T19:14:41,794 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:41,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52122 deadline: 1733339741791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:41,795 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:41,795 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:41,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52066 deadline: 1733339741791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:41,795 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:41,795 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:41,795 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:41,795 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:41,795 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:41,795 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:41,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52088 deadline: 1733339741791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:41,795 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:41,795 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:41,795 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:41,795 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:41,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52124 deadline: 1733339741791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:41,796 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:41,796 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:41,796 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:41,796 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:41,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52102 deadline: 1733339741792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:41,796 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:41,796 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:41,796 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:41,834 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:41,838 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204e546894f9b80495c95622f5058babb50_6fbcb032a86ef6909d8f2378183195f0 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204e546894f9b80495c95622f5058babb50_6fbcb032a86ef6909d8f2378183195f0 2024-12-04T19:14:41,839 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/A/aeae013864ed4f4a99e4109dbefba1ec, store: [table=TestAcidGuarantees family=A region=6fbcb032a86ef6909d8f2378183195f0] 2024-12-04T19:14:41,840 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/A/aeae013864ed4f4a99e4109dbefba1ec is 175, key is test_row_0/A:col10/1733339681415/Put/seqid=0 2024-12-04T19:14:41,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742031_1207 (size=48439) 2024-12-04T19:14:41,882 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-04T19:14:41,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:41,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. as already flushing 2024-12-04T19:14:41,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:41,882 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:41,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:41,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:42,034 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-04T19:14:42,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:42,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. as already flushing 2024-12-04T19:14:42,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:42,035 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:42,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:42,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:42,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-04T19:14:42,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:42,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52122 deadline: 1733339742101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:42,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:42,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52088 deadline: 1733339742101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:42,102 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:42,102 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:42,102 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:42,102 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:42,102 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:42,102 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:42,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:42,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52066 deadline: 1733339742101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:42,102 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:42,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:42,102 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:42,103 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:42,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52102 deadline: 1733339742101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:42,103 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:42,103 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:42,103 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:42,103 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:42,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52124 deadline: 1733339742102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:42,103 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:42,103 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:42,103 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:42,186 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-04T19:14:42,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:42,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. as already flushing 2024-12-04T19:14:42,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:42,187 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:42,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:42,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:42,245 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=257, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/A/aeae013864ed4f4a99e4109dbefba1ec 2024-12-04T19:14:42,252 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/B/98a662f73f984b858a5df8d91b32a34e is 50, key is test_row_0/B:col10/1733339681415/Put/seqid=0 2024-12-04T19:14:42,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742032_1208 (size=12201) 2024-12-04T19:14:42,339 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-04T19:14:42,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:42,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. as already flushing 2024-12-04T19:14:42,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:42,339 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:42,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:42,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:42,492 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-04T19:14:42,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:42,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. as already flushing 2024-12-04T19:14:42,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:42,492 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:42,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:42,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:42,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-04T19:14:42,613 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:42,613 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:42,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52122 deadline: 1733339742611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:42,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52088 deadline: 1733339742611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:42,614 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:42,614 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:42,614 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:42,614 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:42,614 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:42,614 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:42,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52066 deadline: 1733339742612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:42,614 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:42,614 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:42,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52102 deadline: 1733339742612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:42,614 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:42,614 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:42,614 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:42,614 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:42,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52124 deadline: 1733339742612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:42,614 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:42,614 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:42,614 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:42,614 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:42,614 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:42,614 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:42,645 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-04T19:14:42,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:42,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. as already flushing 2024-12-04T19:14:42,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:42,645 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:42,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:42,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:42,658 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/B/98a662f73f984b858a5df8d91b32a34e 2024-12-04T19:14:42,665 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/C/c9d22bffd94b4206ba50ccbda3b04aa7 is 50, key is test_row_0/C:col10/1733339681415/Put/seqid=0 2024-12-04T19:14:42,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742033_1209 (size=12201) 2024-12-04T19:14:42,709 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/C/c9d22bffd94b4206ba50ccbda3b04aa7 2024-12-04T19:14:42,713 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/A/aeae013864ed4f4a99e4109dbefba1ec as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/aeae013864ed4f4a99e4109dbefba1ec 2024-12-04T19:14:42,716 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/aeae013864ed4f4a99e4109dbefba1ec, entries=250, sequenceid=257, filesize=47.3 K 2024-12-04T19:14:42,717 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/B/98a662f73f984b858a5df8d91b32a34e as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/98a662f73f984b858a5df8d91b32a34e 2024-12-04T19:14:42,721 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/98a662f73f984b858a5df8d91b32a34e, entries=150, sequenceid=257, filesize=11.9 K 2024-12-04T19:14:42,722 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/C/c9d22bffd94b4206ba50ccbda3b04aa7 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/c9d22bffd94b4206ba50ccbda3b04aa7 2024-12-04T19:14:42,726 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/c9d22bffd94b4206ba50ccbda3b04aa7, entries=150, sequenceid=257, filesize=11.9 K 2024-12-04T19:14:42,728 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 6fbcb032a86ef6909d8f2378183195f0 in 1314ms, sequenceid=257, compaction requested=true 2024-12-04T19:14:42,728 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6fbcb032a86ef6909d8f2378183195f0: 2024-12-04T19:14:42,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6fbcb032a86ef6909d8f2378183195f0:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T19:14:42,728 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T19:14:42,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:14:42,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6fbcb032a86ef6909d8f2378183195f0:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T19:14:42,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:14:42,728 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T19:14:42,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6fbcb032a86ef6909d8f2378183195f0:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T19:14:42,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T19:14:42,729 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49098 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T19:14:42,729 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 150842 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T19:14:42,729 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1541): 6fbcb032a86ef6909d8f2378183195f0/B is initiating minor compaction (all files) 2024-12-04T19:14:42,729 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1541): 6fbcb032a86ef6909d8f2378183195f0/A is initiating minor compaction (all files) 2024-12-04T19:14:42,729 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6fbcb032a86ef6909d8f2378183195f0/B in TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:42,729 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6fbcb032a86ef6909d8f2378183195f0/A in TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:42,729 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/06ab8ea5ec5a4d308fb491f4e2dc3fcb, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/947a0beb7118462e81e5ebc19837dd11, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/10ae7d405d6849ef9f2eb062f40303c1, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/aeae013864ed4f4a99e4109dbefba1ec] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp, totalSize=147.3 K 2024-12-04T19:14:42,729 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/7eec59502ecd4134a1fd60367199d946, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/4f2429a4e66e4ae782dbfaa92de9eabd, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/e16c5b785a4b470d89e7f8f1a69c527b, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/98a662f73f984b858a5df8d91b32a34e] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp, totalSize=47.9 K 2024-12-04T19:14:42,729 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:42,729 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. files: [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/06ab8ea5ec5a4d308fb491f4e2dc3fcb, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/947a0beb7118462e81e5ebc19837dd11, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/10ae7d405d6849ef9f2eb062f40303c1, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/aeae013864ed4f4a99e4109dbefba1ec] 2024-12-04T19:14:42,730 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 7eec59502ecd4134a1fd60367199d946, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1733339676286 2024-12-04T19:14:42,730 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 06ab8ea5ec5a4d308fb491f4e2dc3fcb, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1733339676286 2024-12-04T19:14:42,730 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f2429a4e66e4ae782dbfaa92de9eabd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1733339678453 2024-12-04T19:14:42,730 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 947a0beb7118462e81e5ebc19837dd11, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1733339678453 2024-12-04T19:14:42,730 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting e16c5b785a4b470d89e7f8f1a69c527b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1733339679120 2024-12-04T19:14:42,730 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 10ae7d405d6849ef9f2eb062f40303c1, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1733339679120 2024-12-04T19:14:42,730 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 98a662f73f984b858a5df8d91b32a34e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1733339681299 2024-12-04T19:14:42,731 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting aeae013864ed4f4a99e4109dbefba1ec, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1733339681297 2024-12-04T19:14:42,736 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=6fbcb032a86ef6909d8f2378183195f0] 2024-12-04T19:14:42,738 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120435e5335dd86b4639a4a64225a895274b_6fbcb032a86ef6909d8f2378183195f0 store=[table=TestAcidGuarantees family=A region=6fbcb032a86ef6909d8f2378183195f0] 2024-12-04T19:14:42,739 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6fbcb032a86ef6909d8f2378183195f0#B#compaction#178 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:14:42,739 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120435e5335dd86b4639a4a64225a895274b_6fbcb032a86ef6909d8f2378183195f0, store=[table=TestAcidGuarantees family=A region=6fbcb032a86ef6909d8f2378183195f0] 2024-12-04T19:14:42,740 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120435e5335dd86b4639a4a64225a895274b_6fbcb032a86ef6909d8f2378183195f0 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=6fbcb032a86ef6909d8f2378183195f0] 2024-12-04T19:14:42,740 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/B/aa98a8d251a147b58f93329df905d73a is 50, key is test_row_0/B:col10/1733339681415/Put/seqid=0 2024-12-04T19:14:42,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742034_1210 (size=4469) 2024-12-04T19:14:42,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742035_1211 (size=12781) 2024-12-04T19:14:42,772 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/B/aa98a8d251a147b58f93329df905d73a as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/aa98a8d251a147b58f93329df905d73a 2024-12-04T19:14:42,777 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6fbcb032a86ef6909d8f2378183195f0/B of 6fbcb032a86ef6909d8f2378183195f0 into aa98a8d251a147b58f93329df905d73a(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:14:42,777 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6fbcb032a86ef6909d8f2378183195f0: 2024-12-04T19:14:42,777 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., storeName=6fbcb032a86ef6909d8f2378183195f0/B, priority=12, startTime=1733339682728; duration=0sec 2024-12-04T19:14:42,777 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T19:14:42,778 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6fbcb032a86ef6909d8f2378183195f0:B 2024-12-04T19:14:42,778 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T19:14:42,779 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49098 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T19:14:42,779 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1541): 6fbcb032a86ef6909d8f2378183195f0/C is initiating minor compaction (all files) 2024-12-04T19:14:42,779 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6fbcb032a86ef6909d8f2378183195f0/C in TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:42,779 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/23421e154e064cf6860ce386f6c86dc2, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/d2911bdede90402c9b9f3f019ec01f33, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/7405d71906e0410b88e76248bb6eaf71, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/c9d22bffd94b4206ba50ccbda3b04aa7] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp, totalSize=47.9 K 2024-12-04T19:14:42,780 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 23421e154e064cf6860ce386f6c86dc2, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1733339676286 2024-12-04T19:14:42,780 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting d2911bdede90402c9b9f3f019ec01f33, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1733339678453 2024-12-04T19:14:42,780 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 7405d71906e0410b88e76248bb6eaf71, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1733339679120 2024-12-04T19:14:42,781 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting c9d22bffd94b4206ba50ccbda3b04aa7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1733339681299 2024-12-04T19:14:42,789 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6fbcb032a86ef6909d8f2378183195f0#C#compaction#179 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:14:42,790 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/C/960d4db894c14ec0b77a4a3604c1208c is 50, key is test_row_0/C:col10/1733339681415/Put/seqid=0 2024-12-04T19:14:42,796 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-04T19:14:42,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:42,797 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing 6fbcb032a86ef6909d8f2378183195f0 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-04T19:14:42,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6fbcb032a86ef6909d8f2378183195f0, store=A 2024-12-04T19:14:42,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:14:42,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6fbcb032a86ef6909d8f2378183195f0, store=B 2024-12-04T19:14:42,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:14:42,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6fbcb032a86ef6909d8f2378183195f0, store=C 2024-12-04T19:14:42,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:14:42,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204f1ecb06b0f474160801a1f1a83cf6ded_6fbcb032a86ef6909d8f2378183195f0 is 50, key is test_row_0/A:col10/1733339681417/Put/seqid=0 2024-12-04T19:14:42,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742036_1212 (size=12781) 2024-12-04T19:14:42,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742037_1213 (size=12454) 2024-12-04T19:14:43,152 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6fbcb032a86ef6909d8f2378183195f0#A#compaction#177 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:14:43,152 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/A/1bc8ade46f684580bc94cfd1e1149ce6 is 175, key is test_row_0/A:col10/1733339681415/Put/seqid=0 2024-12-04T19:14:43,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742038_1214 (size=31735) 2024-12-04T19:14:43,221 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/C/960d4db894c14ec0b77a4a3604c1208c as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/960d4db894c14ec0b77a4a3604c1208c 2024-12-04T19:14:43,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:43,232 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204f1ecb06b0f474160801a1f1a83cf6ded_6fbcb032a86ef6909d8f2378183195f0 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204f1ecb06b0f474160801a1f1a83cf6ded_6fbcb032a86ef6909d8f2378183195f0 2024-12-04T19:14:43,233 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6fbcb032a86ef6909d8f2378183195f0/C of 6fbcb032a86ef6909d8f2378183195f0 into 960d4db894c14ec0b77a4a3604c1208c(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:14:43,233 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6fbcb032a86ef6909d8f2378183195f0: 2024-12-04T19:14:43,233 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., storeName=6fbcb032a86ef6909d8f2378183195f0/C, priority=12, startTime=1733339682728; duration=0sec 2024-12-04T19:14:43,233 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:14:43,233 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6fbcb032a86ef6909d8f2378183195f0:C 2024-12-04T19:14:43,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/A/daede8605d454fc6a2857f90e37dca80, store: [table=TestAcidGuarantees family=A region=6fbcb032a86ef6909d8f2378183195f0] 2024-12-04T19:14:43,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/A/daede8605d454fc6a2857f90e37dca80 is 175, key is test_row_0/A:col10/1733339681417/Put/seqid=0 2024-12-04T19:14:43,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742039_1215 (size=31255) 2024-12-04T19:14:43,563 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/A/1bc8ade46f684580bc94cfd1e1149ce6 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/1bc8ade46f684580bc94cfd1e1149ce6 2024-12-04T19:14:43,571 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6fbcb032a86ef6909d8f2378183195f0/A of 6fbcb032a86ef6909d8f2378183195f0 into 1bc8ade46f684580bc94cfd1e1149ce6(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:14:43,571 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6fbcb032a86ef6909d8f2378183195f0: 2024-12-04T19:14:43,571 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., storeName=6fbcb032a86ef6909d8f2378183195f0/A, priority=12, startTime=1733339682728; duration=0sec 2024-12-04T19:14:43,572 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:14:43,572 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6fbcb032a86ef6909d8f2378183195f0:A 2024-12-04T19:14:43,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-04T19:14:43,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(8855): Flush requested on 6fbcb032a86ef6909d8f2378183195f0 2024-12-04T19:14:43,626 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. as already flushing 2024-12-04T19:14:43,638 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=271, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/A/daede8605d454fc6a2857f90e37dca80 2024-12-04T19:14:43,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/B/c042a5ae2a424c57bbb5ecf7b2170df1 is 50, key is test_row_0/B:col10/1733339681417/Put/seqid=0 2024-12-04T19:14:43,649 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:43,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52122 deadline: 1733339743643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:43,649 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:43,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:43,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52102 deadline: 1733339743644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:43,650 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:43,650 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:43,650 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:43,650 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:43,651 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:43,653 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:43,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52124 deadline: 1733339743645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:43,654 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:43,654 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:43,654 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:43,655 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:43,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52066 deadline: 1733339743650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:43,655 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:43,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52088 deadline: 1733339743650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:43,657 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:43,657 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:43,657 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:43,659 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:43,659 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:43,659 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:43,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742040_1216 (size=12301) 2024-12-04T19:14:43,670 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/B/c042a5ae2a424c57bbb5ecf7b2170df1 2024-12-04T19:14:43,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/C/0d9dec93cec147b58564f84105e15252 is 50, key is test_row_0/C:col10/1733339681417/Put/seqid=0 2024-12-04T19:14:43,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742041_1217 (size=12301) 2024-12-04T19:14:43,701 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/C/0d9dec93cec147b58564f84105e15252 2024-12-04T19:14:43,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/A/daede8605d454fc6a2857f90e37dca80 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/daede8605d454fc6a2857f90e37dca80 2024-12-04T19:14:43,723 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/daede8605d454fc6a2857f90e37dca80, entries=150, sequenceid=271, filesize=30.5 K 2024-12-04T19:14:43,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/B/c042a5ae2a424c57bbb5ecf7b2170df1 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/c042a5ae2a424c57bbb5ecf7b2170df1 2024-12-04T19:14:43,736 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/c042a5ae2a424c57bbb5ecf7b2170df1, entries=150, sequenceid=271, filesize=12.0 K 2024-12-04T19:14:43,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/C/0d9dec93cec147b58564f84105e15252 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/0d9dec93cec147b58564f84105e15252 2024-12-04T19:14:43,744 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/0d9dec93cec147b58564f84105e15252, entries=150, sequenceid=271, filesize=12.0 K 2024-12-04T19:14:43,746 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for 6fbcb032a86ef6909d8f2378183195f0 in 949ms, sequenceid=271, compaction requested=false 2024-12-04T19:14:43,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for 6fbcb032a86ef6909d8f2378183195f0: 2024-12-04T19:14:43,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:43,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-12-04T19:14:43,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-12-04T19:14:43,749 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-12-04T19:14:43,749 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3230 sec 2024-12-04T19:14:43,752 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 2.3280 sec 2024-12-04T19:14:43,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(8855): Flush requested on 6fbcb032a86ef6909d8f2378183195f0 2024-12-04T19:14:43,755 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6fbcb032a86ef6909d8f2378183195f0 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-04T19:14:43,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6fbcb032a86ef6909d8f2378183195f0, store=A 2024-12-04T19:14:43,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T19:14:43,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6fbcb032a86ef6909d8f2378183195f0, store=B 2024-12-04T19:14:43,756 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T19:14:43,756 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6fbcb032a86ef6909d8f2378183195f0, store=C 2024-12-04T19:14:43,756 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T19:14:43,767 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412045c31fdacdfa24346b9f16bb7845d40ae_6fbcb032a86ef6909d8f2378183195f0 is 50, key is test_row_0/A:col10/1733339683643/Put/seqid=0 2024-12-04T19:14:43,783 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:43,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52124 deadline: 1733339743768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:43,783 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:43,784 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:43,784 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:43,784 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:43,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52102 deadline: 1733339743780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:43,784 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:43,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52122 deadline: 1733339743781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:43,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:43,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:43,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:43,785 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:43,785 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:43,785 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:43,787 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:43,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52088 deadline: 1733339743783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:43,788 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:43,788 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:43,788 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:43,788 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:43,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52066 deadline: 1733339743783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:43,788 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:43,788 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:43,788 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:43,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742042_1218 (size=14994) 2024-12-04T19:14:43,892 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:43,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52124 deadline: 1733339743891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:43,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:43,893 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:43,893 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:43,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52122 deadline: 1733339743891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:43,893 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:43,893 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:43,893 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:43,893 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:43,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:43,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52102 deadline: 1733339743891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:43,894 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:43,894 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:43,894 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:43,894 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:43,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52088 deadline: 1733339743891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:43,894 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:43,894 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:43,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52066 deadline: 1733339743891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:43,894 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:43,894 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:43,894 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:43,894 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:43,894 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:44,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:44,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52124 deadline: 1733339744102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:44,103 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:44,103 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:44,103 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:44,103 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:44,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52122 deadline: 1733339744102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:44,103 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:44,103 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:44,103 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:44,103 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:44,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52102 deadline: 1733339744102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:44,103 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:44,103 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:44,103 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:44,103 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:44,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52088 deadline: 1733339744102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:44,104 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:44,104 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:44,104 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:44,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52066 deadline: 1733339744102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:44,104 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:44,104 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:44,104 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:44,104 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:44,218 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:44,222 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412045c31fdacdfa24346b9f16bb7845d40ae_6fbcb032a86ef6909d8f2378183195f0 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412045c31fdacdfa24346b9f16bb7845d40ae_6fbcb032a86ef6909d8f2378183195f0 2024-12-04T19:14:44,223 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/A/a9d239a9a46b477e91a74255abb6382d, store: [table=TestAcidGuarantees family=A region=6fbcb032a86ef6909d8f2378183195f0] 2024-12-04T19:14:44,223 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/A/a9d239a9a46b477e91a74255abb6382d is 175, key is test_row_0/A:col10/1733339683643/Put/seqid=0 2024-12-04T19:14:44,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742043_1219 (size=39949) 2024-12-04T19:14:44,413 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:44,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52124 deadline: 1733339744412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:44,414 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:44,414 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:44,414 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:44,414 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:44,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52122 deadline: 1733339744412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:44,414 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:44,414 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:44,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52102 deadline: 1733339744412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:44,414 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:44,414 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:44,414 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:44,414 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:44,414 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:44,414 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:44,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52088 deadline: 1733339744412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:44,414 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:44,414 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:44,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52066 deadline: 1733339744412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:44,415 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:44,415 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:44,415 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:44,415 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:44,415 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:44,628 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=294, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/A/a9d239a9a46b477e91a74255abb6382d 2024-12-04T19:14:44,636 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/B/ec711d999c2f4799b48ca45a48599ce2 is 50, key is test_row_0/B:col10/1733339683643/Put/seqid=0 2024-12-04T19:14:44,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742044_1220 (size=12301) 2024-12-04T19:14:44,922 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:44,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52124 deadline: 1733339744922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:44,923 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:44,923 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:44,923 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:44,923 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:44,923 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:44,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52122 deadline: 1733339744922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:44,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52102 deadline: 1733339744922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:44,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:44,923 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:44,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:44,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:44,923 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:44,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52088 deadline: 1733339744922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:44,923 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:44,923 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:44,923 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:44,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52066 deadline: 1733339744922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:44,923 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:44,923 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:44,923 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:44,923 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:44,923 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:44,923 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:45,040 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/B/ec711d999c2f4799b48ca45a48599ce2 2024-12-04T19:14:45,048 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/C/7c57cb73f6734e568c0d8bbc673b1d14 is 50, key is test_row_0/C:col10/1733339683643/Put/seqid=0 2024-12-04T19:14:45,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742045_1221 (size=12301) 2024-12-04T19:14:45,451 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/C/7c57cb73f6734e568c0d8bbc673b1d14 2024-12-04T19:14:45,456 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/A/a9d239a9a46b477e91a74255abb6382d as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/a9d239a9a46b477e91a74255abb6382d 2024-12-04T19:14:45,460 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/a9d239a9a46b477e91a74255abb6382d, entries=200, sequenceid=294, filesize=39.0 K 2024-12-04T19:14:45,461 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/B/ec711d999c2f4799b48ca45a48599ce2 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/ec711d999c2f4799b48ca45a48599ce2 2024-12-04T19:14:45,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,465 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/ec711d999c2f4799b48ca45a48599ce2, entries=150, sequenceid=294, filesize=12.0 K 2024-12-04T19:14:45,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,466 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/C/7c57cb73f6734e568c0d8bbc673b1d14 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/7c57cb73f6734e568c0d8bbc673b1d14 2024-12-04T19:14:45,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,474 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/7c57cb73f6734e568c0d8bbc673b1d14, entries=150, sequenceid=294, filesize=12.0 K 2024-12-04T19:14:45,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,475 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 6fbcb032a86ef6909d8f2378183195f0 in 1720ms, sequenceid=294, compaction requested=true 2024-12-04T19:14:45,475 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6fbcb032a86ef6909d8f2378183195f0: 2024-12-04T19:14:45,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6fbcb032a86ef6909d8f2378183195f0:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T19:14:45,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:14:45,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6fbcb032a86ef6909d8f2378183195f0:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T19:14:45,475 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T19:14:45,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:14:45,475 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T19:14:45,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6fbcb032a86ef6909d8f2378183195f0:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T19:14:45,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T19:14:45,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,476 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37383 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T19:14:45,476 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102939 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T19:14:45,476 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1541): 6fbcb032a86ef6909d8f2378183195f0/A is initiating minor compaction (all files) 2024-12-04T19:14:45,476 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1541): 6fbcb032a86ef6909d8f2378183195f0/B is initiating minor compaction (all files) 2024-12-04T19:14:45,476 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6fbcb032a86ef6909d8f2378183195f0/A in TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:45,476 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6fbcb032a86ef6909d8f2378183195f0/B in TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:45,476 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/aa98a8d251a147b58f93329df905d73a, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/c042a5ae2a424c57bbb5ecf7b2170df1, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/ec711d999c2f4799b48ca45a48599ce2] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp, totalSize=36.5 K 2024-12-04T19:14:45,476 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/1bc8ade46f684580bc94cfd1e1149ce6, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/daede8605d454fc6a2857f90e37dca80, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/a9d239a9a46b477e91a74255abb6382d] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp, totalSize=100.5 K 2024-12-04T19:14:45,476 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:45,477 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. files: [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/1bc8ade46f684580bc94cfd1e1149ce6, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/daede8605d454fc6a2857f90e37dca80, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/a9d239a9a46b477e91a74255abb6382d] 2024-12-04T19:14:45,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,477 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting aa98a8d251a147b58f93329df905d73a, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1733339681299 2024-12-04T19:14:45,477 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1bc8ade46f684580bc94cfd1e1149ce6, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1733339681299 2024-12-04T19:14:45,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,477 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting daede8605d454fc6a2857f90e37dca80, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1733339681417 2024-12-04T19:14:45,477 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting c042a5ae2a424c57bbb5ecf7b2170df1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1733339681417 2024-12-04T19:14:45,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,478 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting ec711d999c2f4799b48ca45a48599ce2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1733339683642 2024-12-04T19:14:45,478 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting a9d239a9a46b477e91a74255abb6382d, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1733339683642 2024-12-04T19:14:45,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,484 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6fbcb032a86ef6909d8f2378183195f0#B#compaction#186 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:14:45,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,485 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/B/097815f341504b389f5f9769111ccc21 is 50, key is test_row_0/B:col10/1733339683643/Put/seqid=0 2024-12-04T19:14:45,486 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=6fbcb032a86ef6909d8f2378183195f0] 2024-12-04T19:14:45,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,502 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241204e31b9eec730d4b49b8363b1c9299a8dd_6fbcb032a86ef6909d8f2378183195f0 store=[table=TestAcidGuarantees family=A region=6fbcb032a86ef6909d8f2378183195f0] 2024-12-04T19:14:45,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,503 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241204e31b9eec730d4b49b8363b1c9299a8dd_6fbcb032a86ef6909d8f2378183195f0, store=[table=TestAcidGuarantees family=A region=6fbcb032a86ef6909d8f2378183195f0] 2024-12-04T19:14:45,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,503 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204e31b9eec730d4b49b8363b1c9299a8dd_6fbcb032a86ef6909d8f2378183195f0 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=6fbcb032a86ef6909d8f2378183195f0] 2024-12-04T19:14:45,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742046_1222 (size=12983) 2024-12-04T19:14:45,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742047_1223 (size=4469) 2024-12-04T19:14:45,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-04T19:14:45,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,592 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T19:14:45,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,594 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T19:14:45,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-12-04T19:14:45,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,597 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T19:14:45,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-04T19:14:45,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,598 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T19:14:45,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,598 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T19:14:45,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-04T19:14:45,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,751 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-04T19:14:45,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:45,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,753 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing 6fbcb032a86ef6909d8f2378183195f0 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-04T19:14:45,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6fbcb032a86ef6909d8f2378183195f0, store=A 2024-12-04T19:14:45,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:14:45,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6fbcb032a86ef6909d8f2378183195f0, store=B 2024-12-04T19:14:45,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:14:45,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6fbcb032a86ef6909d8f2378183195f0, store=C 2024-12-04T19:14:45,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:14:45,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120407f241551bf3424b89da4f683ecfddda_6fbcb032a86ef6909d8f2378183195f0 is 50, key is test_row_1/A:col10/1733339683781/Put/seqid=0 2024-12-04T19:14:45,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742048_1224 (size=9914) 2024-12-04T19:14:45,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,780 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120407f241551bf3424b89da4f683ecfddda_6fbcb032a86ef6909d8f2378183195f0 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120407f241551bf3424b89da4f683ecfddda_6fbcb032a86ef6909d8f2378183195f0 2024-12-04T19:14:45,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/A/8cbd9fd0d08a4840b3ddc5b7799d45f9, store: [table=TestAcidGuarantees family=A region=6fbcb032a86ef6909d8f2378183195f0] 2024-12-04T19:14:45,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/A/8cbd9fd0d08a4840b3ddc5b7799d45f9 is 175, key is test_row_1/A:col10/1733339683781/Put/seqid=0 2024-12-04T19:14:45,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742049_1225 (size=22561) 2024-12-04T19:14:45,790 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=305, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/A/8cbd9fd0d08a4840b3ddc5b7799d45f9 2024-12-04T19:14:45,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/B/14691e53cb7c43d2a6cd2752c74c1666 is 50, key is test_row_1/B:col10/1733339683781/Put/seqid=0 2024-12-04T19:14:45,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742050_1226 (size=9857) 2024-12-04T19:14:45,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,815 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=305 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/B/14691e53cb7c43d2a6cd2752c74c1666 2024-12-04T19:14:45,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/C/d616df55eaeb4e18b0d1d55c3983129d is 50, key is test_row_1/C:col10/1733339683781/Put/seqid=0 2024-12-04T19:14:45,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742051_1227 (size=9857) 2024-12-04T19:14:45,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-04T19:14:45,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,914 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6fbcb032a86ef6909d8f2378183195f0#A#compaction#187 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:14:45,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,914 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/A/d8358b76c4cc435eac754787a222402e is 175, key is test_row_0/A:col10/1733339683643/Put/seqid=0 2024-12-04T19:14:45,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,916 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/B/097815f341504b389f5f9769111ccc21 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/097815f341504b389f5f9769111ccc21 2024-12-04T19:14:45,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,922 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6fbcb032a86ef6909d8f2378183195f0/B of 6fbcb032a86ef6909d8f2378183195f0 into 097815f341504b389f5f9769111ccc21(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:14:45,922 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6fbcb032a86ef6909d8f2378183195f0: 2024-12-04T19:14:45,922 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., storeName=6fbcb032a86ef6909d8f2378183195f0/B, priority=13, startTime=1733339685475; duration=0sec 2024-12-04T19:14:45,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,922 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T19:14:45,922 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6fbcb032a86ef6909d8f2378183195f0:B 2024-12-04T19:14:45,923 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T19:14:45,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,924 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37383 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T19:14:45,924 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1541): 6fbcb032a86ef6909d8f2378183195f0/C is initiating minor compaction (all files) 2024-12-04T19:14:45,924 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6fbcb032a86ef6909d8f2378183195f0/C in TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:45,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,924 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/960d4db894c14ec0b77a4a3604c1208c, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/0d9dec93cec147b58564f84105e15252, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/7c57cb73f6734e568c0d8bbc673b1d14] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp, totalSize=36.5 K 2024-12-04T19:14:45,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,925 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 960d4db894c14ec0b77a4a3604c1208c, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1733339681299 2024-12-04T19:14:45,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,925 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d9dec93cec147b58564f84105e15252, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1733339681417 2024-12-04T19:14:45,925 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c57cb73f6734e568c0d8bbc673b1d14, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1733339683642 2024-12-04T19:14:45,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(8855): Flush requested on 6fbcb032a86ef6909d8f2378183195f0 2024-12-04T19:14:45,934 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. as already flushing 2024-12-04T19:14:45,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742052_1228 (size=31937) 2024-12-04T19:14:45,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,938 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6fbcb032a86ef6909d8f2378183195f0#C#compaction#191 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:14:45,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,938 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/C/e8e07769cd32443ab3f4e000d227a0f4 is 50, key is test_row_0/C:col10/1733339683643/Put/seqid=0 2024-12-04T19:14:45,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742053_1229 (size=12983) 2024-12-04T19:14:45,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,953 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/C/e8e07769cd32443ab3f4e000d227a0f4 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/e8e07769cd32443ab3f4e000d227a0f4 2024-12-04T19:14:45,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,962 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6fbcb032a86ef6909d8f2378183195f0/C of 6fbcb032a86ef6909d8f2378183195f0 into e8e07769cd32443ab3f4e000d227a0f4(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:14:45,963 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6fbcb032a86ef6909d8f2378183195f0: 2024-12-04T19:14:45,963 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., storeName=6fbcb032a86ef6909d8f2378183195f0/C, priority=13, startTime=1733339685475; duration=0sec 2024-12-04T19:14:45,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,963 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:14:45,963 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6fbcb032a86ef6909d8f2378183195f0:C 2024-12-04T19:14:45,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:45,986 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:45,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52124 deadline: 1733339745980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:45,986 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733339685958, max timestamp=1733339685984Num uniques 859; 2024-12-04T19:14:45,986 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-04T19:14:45,986 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:45,986 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:45,987 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:45,993 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:45,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52122 deadline: 1733339745988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:45,994 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:45,994 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:45,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52102 deadline: 1733339745989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:45,994 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:45,994 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:45,994 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:45,994 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:45,994 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:45,996 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:45,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52088 deadline: 1733339745991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:45,996 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:45,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52066 deadline: 1733339745993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:45,998 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:45,998 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:45,998 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:45,998 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:45,999 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:45,999 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:46,094 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:46,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52124 deadline: 1733339746091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:46,095 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:46,095 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:46,095 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:46,118 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:46,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52102 deadline: 1733339746116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:46,119 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:46,119 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:46,119 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:46,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52088 deadline: 1733339746116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:46,119 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:46,119 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:46,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52122 deadline: 1733339746117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:46,119 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:46,119 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:46,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:46,119 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:46,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52066 deadline: 1733339746117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:46,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:46,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:46,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:46,120 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:46,120 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:46,120 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:46,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-04T19:14:46,260 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=305 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/C/d616df55eaeb4e18b0d1d55c3983129d 2024-12-04T19:14:46,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/A/8cbd9fd0d08a4840b3ddc5b7799d45f9 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/8cbd9fd0d08a4840b3ddc5b7799d45f9 2024-12-04T19:14:46,269 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/8cbd9fd0d08a4840b3ddc5b7799d45f9, entries=100, sequenceid=305, filesize=22.0 K 2024-12-04T19:14:46,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/B/14691e53cb7c43d2a6cd2752c74c1666 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/14691e53cb7c43d2a6cd2752c74c1666 2024-12-04T19:14:46,279 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/14691e53cb7c43d2a6cd2752c74c1666, entries=100, sequenceid=305, filesize=9.6 K 2024-12-04T19:14:46,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/C/d616df55eaeb4e18b0d1d55c3983129d as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/d616df55eaeb4e18b0d1d55c3983129d 2024-12-04T19:14:46,285 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/d616df55eaeb4e18b0d1d55c3983129d, entries=100, sequenceid=305, filesize=9.6 K 2024-12-04T19:14:46,290 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=123.00 KB/125950 for 6fbcb032a86ef6909d8f2378183195f0 in 534ms, sequenceid=305, compaction requested=false 2024-12-04T19:14:46,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for 6fbcb032a86ef6909d8f2378183195f0: 2024-12-04T19:14:46,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:46,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-12-04T19:14:46,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-12-04T19:14:46,294 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-12-04T19:14:46,294 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 694 msec 2024-12-04T19:14:46,296 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 701 msec 2024-12-04T19:14:46,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(8855): Flush requested on 6fbcb032a86ef6909d8f2378183195f0 2024-12-04T19:14:46,303 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6fbcb032a86ef6909d8f2378183195f0 3/3 column families, dataSize=129.71 KB heapSize=376.95 KB 2024-12-04T19:14:46,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6fbcb032a86ef6909d8f2378183195f0, store=A 2024-12-04T19:14:46,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T19:14:46,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6fbcb032a86ef6909d8f2378183195f0, store=B 2024-12-04T19:14:46,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T19:14:46,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6fbcb032a86ef6909d8f2378183195f0, store=C 2024-12-04T19:14:46,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T19:14:46,312 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120407e5cf1637fc4771a0e2a6c37d528d13_6fbcb032a86ef6909d8f2378183195f0 is 50, key is test_row_0/A:col10/1733339685991/Put/seqid=0 2024-12-04T19:14:46,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742054_1230 (size=14994) 2024-12-04T19:14:46,321 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:46,325 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120407e5cf1637fc4771a0e2a6c37d528d13_6fbcb032a86ef6909d8f2378183195f0 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120407e5cf1637fc4771a0e2a6c37d528d13_6fbcb032a86ef6909d8f2378183195f0 2024-12-04T19:14:46,326 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/A/8f445a09ee8049a6a746b7837d24cb48, store: [table=TestAcidGuarantees family=A region=6fbcb032a86ef6909d8f2378183195f0] 2024-12-04T19:14:46,326 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/A/8f445a09ee8049a6a746b7837d24cb48 is 175, key is test_row_0/A:col10/1733339685991/Put/seqid=0 2024-12-04T19:14:46,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742055_1231 (size=39949) 2024-12-04T19:14:46,340 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/A/d8358b76c4cc435eac754787a222402e as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/d8358b76c4cc435eac754787a222402e 2024-12-04T19:14:46,346 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6fbcb032a86ef6909d8f2378183195f0/A of 6fbcb032a86ef6909d8f2378183195f0 into d8358b76c4cc435eac754787a222402e(size=31.2 K), total size for store is 53.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:14:46,346 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6fbcb032a86ef6909d8f2378183195f0: 2024-12-04T19:14:46,346 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., storeName=6fbcb032a86ef6909d8f2378183195f0/A, priority=13, startTime=1733339685475; duration=0sec 2024-12-04T19:14:46,347 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:14:46,347 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6fbcb032a86ef6909d8f2378183195f0:A 2024-12-04T19:14:46,353 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:46,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52124 deadline: 1733339746347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:46,354 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:46,354 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:46,354 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:46,354 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:46,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52102 deadline: 1733339746347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:46,355 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:46,355 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:46,355 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:46,355 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:46,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52088 deadline: 1733339746348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:46,355 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:46,355 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:46,355 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:46,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52122 deadline: 1733339746349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:46,355 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:46,355 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:46,355 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:46,355 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:46,355 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:46,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52066 deadline: 1733339746350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:46,356 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:46,356 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:46,356 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:46,463 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:46,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52124 deadline: 1733339746462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:46,464 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:46,464 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:46,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52102 deadline: 1733339746462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:46,464 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:46,464 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:46,464 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:46,464 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:46,464 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:46,464 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:46,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52088 deadline: 1733339746462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:46,464 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:46,464 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:46,464 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:46,464 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:46,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52122 deadline: 1733339746462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:46,465 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:46,465 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:46,465 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:46,465 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:46,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52066 deadline: 1733339746462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:46,465 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:46,465 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:46,465 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:46,675 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:46,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52124 deadline: 1733339746672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:46,676 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:46,676 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:46,676 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:46,676 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:46,676 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:46,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52102 deadline: 1733339746672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:46,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52088 deadline: 1733339746672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:46,676 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:46,676 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:46,676 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:46,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52122 deadline: 1733339746672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:46,676 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:46,676 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:46,676 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:46,676 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:46,676 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:46,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52066 deadline: 1733339746672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:46,676 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:46,676 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:46,676 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:46,676 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:46,676 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:46,676 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:46,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-04T19:14:46,732 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T19:14:46,732 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T19:14:46,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-12-04T19:14:46,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-04T19:14:46,734 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T19:14:46,734 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T19:14:46,735 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T19:14:46,737 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=333, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/A/8f445a09ee8049a6a746b7837d24cb48 2024-12-04T19:14:46,744 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/B/acff263634a74f949bf45a8de0ba0937 is 50, key is test_row_0/B:col10/1733339685991/Put/seqid=0 2024-12-04T19:14:46,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742056_1232 (size=12301) 2024-12-04T19:14:46,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-04T19:14:46,886 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-04T19:14:46,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:46,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. as already flushing 2024-12-04T19:14:46,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:46,887 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:46,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:46,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:46,986 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:46,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52124 deadline: 1733339746982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:46,986 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:46,986 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:46,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52102 deadline: 1733339746982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:46,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52088 deadline: 1733339746982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:46,987 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:46,987 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:46,987 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:46,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52122 deadline: 1733339746982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:46,987 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:46,987 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:46,987 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:46,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52066 deadline: 1733339746982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:46,987 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:46,987 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:46,987 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:46,987 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:46,987 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:46,987 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:46,987 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:46,987 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:46,987 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:46,987 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:46,987 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:47,039 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-04T19:14:47,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:47,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. as already flushing 2024-12-04T19:14:47,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:47,040 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:47,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:47,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:47,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-04T19:14:47,148 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/B/acff263634a74f949bf45a8de0ba0937 2024-12-04T19:14:47,154 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/C/72ae60f0031143caa84748b3ef99af38 is 50, key is test_row_0/C:col10/1733339685991/Put/seqid=0 2024-12-04T19:14:47,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742057_1233 (size=12301) 2024-12-04T19:14:47,158 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/C/72ae60f0031143caa84748b3ef99af38 2024-12-04T19:14:47,162 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/A/8f445a09ee8049a6a746b7837d24cb48 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/8f445a09ee8049a6a746b7837d24cb48 2024-12-04T19:14:47,165 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/8f445a09ee8049a6a746b7837d24cb48, entries=200, sequenceid=333, filesize=39.0 K 2024-12-04T19:14:47,166 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/B/acff263634a74f949bf45a8de0ba0937 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/acff263634a74f949bf45a8de0ba0937 2024-12-04T19:14:47,170 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/acff263634a74f949bf45a8de0ba0937, entries=150, sequenceid=333, filesize=12.0 K 2024-12-04T19:14:47,171 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/C/72ae60f0031143caa84748b3ef99af38 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/72ae60f0031143caa84748b3ef99af38 2024-12-04T19:14:47,175 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/72ae60f0031143caa84748b3ef99af38, entries=150, sequenceid=333, filesize=12.0 K 2024-12-04T19:14:47,175 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~129.71 KB/132820, heapSize ~376.91 KB/385952, currentSize=46.96 KB/48090 for 6fbcb032a86ef6909d8f2378183195f0 in 872ms, sequenceid=333, compaction requested=true 2024-12-04T19:14:47,175 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6fbcb032a86ef6909d8f2378183195f0: 2024-12-04T19:14:47,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6fbcb032a86ef6909d8f2378183195f0:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T19:14:47,176 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T19:14:47,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:14:47,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6fbcb032a86ef6909d8f2378183195f0:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T19:14:47,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T19:14:47,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6fbcb032a86ef6909d8f2378183195f0:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T19:14:47,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-04T19:14:47,176 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T19:14:47,177 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94447 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T19:14:47,177 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1541): 6fbcb032a86ef6909d8f2378183195f0/A is initiating minor compaction (all files) 2024-12-04T19:14:47,177 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T19:14:47,177 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6fbcb032a86ef6909d8f2378183195f0/A in TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:47,177 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1541): 6fbcb032a86ef6909d8f2378183195f0/B is initiating minor compaction (all files) 2024-12-04T19:14:47,177 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6fbcb032a86ef6909d8f2378183195f0/B in TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:47,177 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/d8358b76c4cc435eac754787a222402e, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/8cbd9fd0d08a4840b3ddc5b7799d45f9, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/8f445a09ee8049a6a746b7837d24cb48] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp, totalSize=92.2 K 2024-12-04T19:14:47,177 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:47,177 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/097815f341504b389f5f9769111ccc21, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/14691e53cb7c43d2a6cd2752c74c1666, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/acff263634a74f949bf45a8de0ba0937] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp, totalSize=34.3 K 2024-12-04T19:14:47,177 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. files: [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/d8358b76c4cc435eac754787a222402e, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/8cbd9fd0d08a4840b3ddc5b7799d45f9, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/8f445a09ee8049a6a746b7837d24cb48] 2024-12-04T19:14:47,177 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 097815f341504b389f5f9769111ccc21, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1733339683642 2024-12-04T19:14:47,177 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting d8358b76c4cc435eac754787a222402e, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1733339683642 2024-12-04T19:14:47,178 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 14691e53cb7c43d2a6cd2752c74c1666, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1733339683781 2024-12-04T19:14:47,178 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8cbd9fd0d08a4840b3ddc5b7799d45f9, keycount=100, bloomtype=ROW, size=22.0 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1733339683781 2024-12-04T19:14:47,178 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting acff263634a74f949bf45a8de0ba0937, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1733339685988 2024-12-04T19:14:47,178 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8f445a09ee8049a6a746b7837d24cb48, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1733339685986 2024-12-04T19:14:47,183 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=6fbcb032a86ef6909d8f2378183195f0] 2024-12-04T19:14:47,185 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6fbcb032a86ef6909d8f2378183195f0#B#compaction#195 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:14:47,185 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412041e01f01cca7c42c1a9269cc310ddadc3_6fbcb032a86ef6909d8f2378183195f0 store=[table=TestAcidGuarantees family=A region=6fbcb032a86ef6909d8f2378183195f0] 2024-12-04T19:14:47,185 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/B/d984aac7c9e14133a920c6878e0b981b is 50, key is test_row_0/B:col10/1733339685991/Put/seqid=0 2024-12-04T19:14:47,186 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412041e01f01cca7c42c1a9269cc310ddadc3_6fbcb032a86ef6909d8f2378183195f0, store=[table=TestAcidGuarantees family=A region=6fbcb032a86ef6909d8f2378183195f0] 2024-12-04T19:14:47,187 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412041e01f01cca7c42c1a9269cc310ddadc3_6fbcb032a86ef6909d8f2378183195f0 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=6fbcb032a86ef6909d8f2378183195f0] 2024-12-04T19:14:47,192 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-04T19:14:47,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742058_1234 (size=13085) 2024-12-04T19:14:47,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:47,192 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing 6fbcb032a86ef6909d8f2378183195f0 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-04T19:14:47,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6fbcb032a86ef6909d8f2378183195f0, store=A 2024-12-04T19:14:47,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:14:47,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6fbcb032a86ef6909d8f2378183195f0, store=B 2024-12-04T19:14:47,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:14:47,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6fbcb032a86ef6909d8f2378183195f0, store=C 2024-12-04T19:14:47,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:14:47,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742059_1235 (size=4469) 2024-12-04T19:14:47,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412040f3c2ea3f091409ab9fb25fc9831adaf_6fbcb032a86ef6909d8f2378183195f0 is 50, key is test_row_0/A:col10/1733339686326/Put/seqid=0 2024-12-04T19:14:47,202 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6fbcb032a86ef6909d8f2378183195f0#A#compaction#196 average throughput is 1.36 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:14:47,203 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/A/f02e6be1b5b74b549fcda387394a9f7c is 175, key is test_row_0/A:col10/1733339685991/Put/seqid=0 2024-12-04T19:14:47,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742060_1236 (size=12454) 2024-12-04T19:14:47,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742061_1237 (size=32039) 2024-12-04T19:14:47,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-04T19:14:47,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(8855): Flush requested on 6fbcb032a86ef6909d8f2378183195f0 2024-12-04T19:14:47,492 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. as already flushing 2024-12-04T19:14:47,513 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1733339687502, max timestamp=1733339687511Num uniques 859; 2024-12-04T19:14:47,513 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-12-04T19:14:47,514 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1733339687502, max timestamp=1733339687511Num uniques 859; 2024-12-04T19:14:47,514 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1733339687502, max timestamp=1733339687511Num uniques 859; 2024-12-04T19:14:47,514 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-12-04T19:14:47,514 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-12-04T19:14:47,527 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:47,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:47,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52124 deadline: 1733339747520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:47,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52088 deadline: 1733339747521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:47,527 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:47,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52066 deadline: 1733339747522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:47,527 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:47,527 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:47,527 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:47,527 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:47,527 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:47,527 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:47,527 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:47,527 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:47,527 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:47,530 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:47,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52122 deadline: 1733339747527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:47,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:47,530 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:47,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:47,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:47,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52102 deadline: 1733339747527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:47,531 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:47,531 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:47,531 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:47,596 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/B/d984aac7c9e14133a920c6878e0b981b as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/d984aac7c9e14133a920c6878e0b981b 2024-12-04T19:14:47,601 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6fbcb032a86ef6909d8f2378183195f0/B of 6fbcb032a86ef6909d8f2378183195f0 into d984aac7c9e14133a920c6878e0b981b(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:14:47,601 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6fbcb032a86ef6909d8f2378183195f0: 2024-12-04T19:14:47,601 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., storeName=6fbcb032a86ef6909d8f2378183195f0/B, priority=13, startTime=1733339687176; duration=0sec 2024-12-04T19:14:47,601 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T19:14:47,601 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6fbcb032a86ef6909d8f2378183195f0:B 2024-12-04T19:14:47,601 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T19:14:47,602 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T19:14:47,602 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1541): 6fbcb032a86ef6909d8f2378183195f0/C is initiating minor compaction (all files) 2024-12-04T19:14:47,602 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6fbcb032a86ef6909d8f2378183195f0/C in TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:47,602 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/e8e07769cd32443ab3f4e000d227a0f4, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/d616df55eaeb4e18b0d1d55c3983129d, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/72ae60f0031143caa84748b3ef99af38] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp, totalSize=34.3 K 2024-12-04T19:14:47,603 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting e8e07769cd32443ab3f4e000d227a0f4, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1733339683642 2024-12-04T19:14:47,603 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting d616df55eaeb4e18b0d1d55c3983129d, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1733339683781 2024-12-04T19:14:47,603 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 72ae60f0031143caa84748b3ef99af38, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1733339685988 2024-12-04T19:14:47,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:47,608 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412040f3c2ea3f091409ab9fb25fc9831adaf_6fbcb032a86ef6909d8f2378183195f0 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412040f3c2ea3f091409ab9fb25fc9831adaf_6fbcb032a86ef6909d8f2378183195f0 2024-12-04T19:14:47,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/A/0813e33b19d14c3d94bc5ae8a4f5e476, store: [table=TestAcidGuarantees family=A region=6fbcb032a86ef6909d8f2378183195f0] 2024-12-04T19:14:47,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/A/0813e33b19d14c3d94bc5ae8a4f5e476 is 175, key is test_row_0/A:col10/1733339686326/Put/seqid=0 2024-12-04T19:14:47,611 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6fbcb032a86ef6909d8f2378183195f0#C#compaction#198 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:14:47,611 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/C/4e6383bf26ad4bd6b3a58ceb9d4b908d is 50, key is test_row_0/C:col10/1733339685991/Put/seqid=0 2024-12-04T19:14:47,613 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/A/f02e6be1b5b74b549fcda387394a9f7c as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/f02e6be1b5b74b549fcda387394a9f7c 2024-12-04T19:14:47,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742062_1238 (size=31255) 2024-12-04T19:14:47,620 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6fbcb032a86ef6909d8f2378183195f0/A of 6fbcb032a86ef6909d8f2378183195f0 into f02e6be1b5b74b549fcda387394a9f7c(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:14:47,620 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6fbcb032a86ef6909d8f2378183195f0: 2024-12-04T19:14:47,620 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., storeName=6fbcb032a86ef6909d8f2378183195f0/A, priority=13, startTime=1733339687176; duration=0sec 2024-12-04T19:14:47,620 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:14:47,620 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6fbcb032a86ef6909d8f2378183195f0:A 2024-12-04T19:14:47,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742063_1239 (size=13085) 2024-12-04T19:14:47,628 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/C/4e6383bf26ad4bd6b3a58ceb9d4b908d as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/4e6383bf26ad4bd6b3a58ceb9d4b908d 2024-12-04T19:14:47,632 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6fbcb032a86ef6909d8f2378183195f0/C of 6fbcb032a86ef6909d8f2378183195f0 into 4e6383bf26ad4bd6b3a58ceb9d4b908d(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:14:47,632 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6fbcb032a86ef6909d8f2378183195f0: 2024-12-04T19:14:47,632 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., storeName=6fbcb032a86ef6909d8f2378183195f0/C, priority=13, startTime=1733339687176; duration=0sec 2024-12-04T19:14:47,633 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:14:47,633 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6fbcb032a86ef6909d8f2378183195f0:C 2024-12-04T19:14:47,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:47,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52088 deadline: 1733339747632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:47,634 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:47,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:47,634 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:47,634 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:47,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52124 deadline: 1733339747632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:47,635 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:47,635 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:47,635 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:47,635 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:47,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52066 deadline: 1733339747632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:47,635 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:47,635 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:47,635 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:47,645 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:47,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52122 deadline: 1733339747641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:47,646 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:47,646 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:47,646 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:47,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52102 deadline: 1733339747641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:47,646 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:47,647 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:47,647 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:47,647 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:47,846 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:47,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52088 deadline: 1733339747842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:47,846 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:47,846 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:47,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52124 deadline: 1733339747842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:47,846 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:47,847 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:47,847 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:47,847 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:47,847 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:47,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52066 deadline: 1733339747842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:47,847 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:47,847 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:47,847 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:47,847 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:47,855 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:47,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52122 deadline: 1733339747851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:47,855 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:47,855 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:47,855 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:47,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52102 deadline: 1733339747851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:47,855 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:47,855 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:47,855 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:47,855 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:47,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-04T19:14:48,014 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-04T19:14:48,017 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=344, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/A/0813e33b19d14c3d94bc5ae8a4f5e476 2024-12-04T19:14:48,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/B/efe980b4c3e34779bad7af9823d9e368 is 50, key is test_row_0/B:col10/1733339686326/Put/seqid=0 2024-12-04T19:14:48,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742064_1240 (size=12301) 2024-12-04T19:14:48,153 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:48,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52088 deadline: 1733339748152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:48,154 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:48,154 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:48,154 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:48,154 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:48,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52124 deadline: 1733339748152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:48,154 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:48,154 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:48,154 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:48,154 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:48,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52066 deadline: 1733339748152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:48,155 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:48,155 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:48,155 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:48,162 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:48,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52122 deadline: 1733339748161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:48,163 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:48,163 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:48,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52102 deadline: 1733339748161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:48,163 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:48,163 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:48,163 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:48,163 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:48,163 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:48,430 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=344 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/B/efe980b4c3e34779bad7af9823d9e368 2024-12-04T19:14:48,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/C/00f847659b1548f2b74ad06a26d41a99 is 50, key is test_row_0/C:col10/1733339686326/Put/seqid=0 2024-12-04T19:14:48,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742065_1241 (size=12301) 2024-12-04T19:14:48,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:48,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52088 deadline: 1733339748662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:48,664 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:48,664 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:48,664 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:48,664 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:48,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52124 deadline: 1733339748662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:48,664 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:48,664 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:48,664 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:48,664 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:48,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52066 deadline: 1733339748662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:48,665 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:48,665 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:48,665 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:48,672 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:48,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52122 deadline: 1733339748671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:48,672 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:48,672 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:48,672 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:48,672 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:48,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52102 deadline: 1733339748671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:48,673 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:48,673 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:48,673 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:48,845 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=344 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/C/00f847659b1548f2b74ad06a26d41a99 2024-12-04T19:14:48,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/A/0813e33b19d14c3d94bc5ae8a4f5e476 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/0813e33b19d14c3d94bc5ae8a4f5e476 2024-12-04T19:14:48,852 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/0813e33b19d14c3d94bc5ae8a4f5e476, entries=150, sequenceid=344, filesize=30.5 K 2024-12-04T19:14:48,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/B/efe980b4c3e34779bad7af9823d9e368 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/efe980b4c3e34779bad7af9823d9e368 2024-12-04T19:14:48,857 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/efe980b4c3e34779bad7af9823d9e368, entries=150, sequenceid=344, filesize=12.0 K 2024-12-04T19:14:48,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/C/00f847659b1548f2b74ad06a26d41a99 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/00f847659b1548f2b74ad06a26d41a99 2024-12-04T19:14:48,869 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/00f847659b1548f2b74ad06a26d41a99, entries=150, sequenceid=344, filesize=12.0 K 2024-12-04T19:14:48,870 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for 6fbcb032a86ef6909d8f2378183195f0 in 1677ms, sequenceid=344, compaction requested=false 2024-12-04T19:14:48,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for 6fbcb032a86ef6909d8f2378183195f0: 2024-12-04T19:14:48,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:48,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-12-04T19:14:48,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-12-04T19:14:48,873 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-12-04T19:14:48,873 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1360 sec 2024-12-04T19:14:48,874 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 2.1410 sec 2024-12-04T19:14:48,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-04T19:14:48,892 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T19:14:48,893 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T19:14:48,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2024-12-04T19:14:48,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-04T19:14:48,894 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T19:14:48,895 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T19:14:48,895 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T19:14:49,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-04T19:14:49,047 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-12-04T19:14:49,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:49,047 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing 6fbcb032a86ef6909d8f2378183195f0 3/3 column families, dataSize=140.89 KB heapSize=381.47 KB 2024-12-04T19:14:49,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6fbcb032a86ef6909d8f2378183195f0, store=A 2024-12-04T19:14:49,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T19:14:49,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6fbcb032a86ef6909d8f2378183195f0, store=B 2024-12-04T19:14:49,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T19:14:49,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6fbcb032a86ef6909d8f2378183195f0, store=C 2024-12-04T19:14:49,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T19:14:49,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204b914aa7d61fd4811bb9b59185929fed9_6fbcb032a86ef6909d8f2378183195f0 is 50, key is test_row_0/A:col10/1733339687521/Put/seqid=0 2024-12-04T19:14:49,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742066_1242 (size=12454) 2024-12-04T19:14:49,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-04T19:14:49,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:49,462 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204b914aa7d61fd4811bb9b59185929fed9_6fbcb032a86ef6909d8f2378183195f0 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204b914aa7d61fd4811bb9b59185929fed9_6fbcb032a86ef6909d8f2378183195f0 2024-12-04T19:14:49,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/A/95c2cd46f96149e2af0606b75f313325, store: [table=TestAcidGuarantees family=A region=6fbcb032a86ef6909d8f2378183195f0] 2024-12-04T19:14:49,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/A/95c2cd46f96149e2af0606b75f313325 is 175, key is test_row_0/A:col10/1733339687521/Put/seqid=0 2024-12-04T19:14:49,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742067_1243 (size=31255) 2024-12-04T19:14:49,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-04T19:14:49,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(8855): Flush requested on 6fbcb032a86ef6909d8f2378183195f0 2024-12-04T19:14:49,673 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. as already flushing 2024-12-04T19:14:49,680 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:49,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52124 deadline: 1733339749678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:49,680 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:49,680 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:49,680 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:49,683 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:49,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52066 deadline: 1733339749679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:49,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:49,684 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:49,684 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:49,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52088 deadline: 1733339749680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:49,684 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:49,684 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:49,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:49,684 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:49,684 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:49,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52122 deadline: 1733339749681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:49,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:49,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52102 deadline: 1733339749681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:49,684 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 9ba2734cd39f,33969,1733339629791 for 'test_row_2' in 6fbcb032a86ef6909d8f2378183195f0 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2157 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-04T19:14:49,684 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 9ba2734cd39f,33969,1733339629791 for 'test_row_0' in 6fbcb032a86ef6909d8f2378183195f0 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2157 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-04T19:14:49,685 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:49,685 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:49,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:49,685 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:49,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:49,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:49,794 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:49,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52124 deadline: 1733339749791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:49,794 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:49,794 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:49,794 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:49,794 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:49,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52066 deadline: 1733339749792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:49,795 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:49,795 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:49,795 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:49,795 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:49,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52088 deadline: 1733339749792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:49,795 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:49,795 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:49,795 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:49,868 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=380, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/A/95c2cd46f96149e2af0606b75f313325 2024-12-04T19:14:49,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/B/d39ce156f19f4d8d97d5a2a1f2979d41 is 50, key is test_row_0/B:col10/1733339687521/Put/seqid=0 2024-12-04T19:14:49,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742068_1244 (size=12301) 2024-12-04T19:14:50,002 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:50,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52124 deadline: 1733339750001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:50,003 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:50,003 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:50,003 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:50,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52066 deadline: 1733339750001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:50,003 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:50,003 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:50,003 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:50,003 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:50,003 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:50,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52088 deadline: 1733339750001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:50,003 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:50,003 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:50,003 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:50,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-04T19:14:50,175 INFO [Thread-707 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-707. 2024-12-04T19:14:50,175 DEBUG [Thread-707 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T19:14:50,175 DEBUG [Thread-707 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:14:50,176 DEBUG [Thread-707 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:14:50,176 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T19:14:50,180 INFO [Thread-711 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-711. 2024-12-04T19:14:50,180 DEBUG [Thread-711 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T19:14:50,180 DEBUG [Thread-711 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:14:50,180 DEBUG [Thread-711 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:14:50,180 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T19:14:50,180 INFO [Thread-709 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-709. 2024-12-04T19:14:50,180 DEBUG [Thread-709 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T19:14:50,180 DEBUG [Thread-709 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:14:50,181 DEBUG [Thread-709 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:14:50,181 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T19:14:50,181 INFO [Thread-713 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-713. 2024-12-04T19:14:50,181 DEBUG [Thread-713 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T19:14:50,181 DEBUG [Thread-713 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:14:50,181 DEBUG [Thread-713 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:14:50,181 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T19:14:50,284 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/B/d39ce156f19f4d8d97d5a2a1f2979d41 2024-12-04T19:14:50,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/C/2a98a46de8ec430ebc60b7f5b086f1ad is 50, key is test_row_0/C:col10/1733339687521/Put/seqid=0 2024-12-04T19:14:50,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742069_1245 (size=12301) 2024-12-04T19:14:50,312 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:50,312 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:50,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52124 deadline: 1733339750311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:50,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52066 deadline: 1733339750312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:50,312 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:50,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52088 deadline: 1733339750312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:50,312 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:50,312 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:50,312 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:50,312 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:50,312 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:50,312 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:50,312 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:50,312 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:50,312 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:14:50,706 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/C/2a98a46de8ec430ebc60b7f5b086f1ad 2024-12-04T19:14:50,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/A/95c2cd46f96149e2af0606b75f313325 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/95c2cd46f96149e2af0606b75f313325 2024-12-04T19:14:50,721 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/95c2cd46f96149e2af0606b75f313325, entries=150, sequenceid=380, filesize=30.5 K 2024-12-04T19:14:50,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/B/d39ce156f19f4d8d97d5a2a1f2979d41 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/d39ce156f19f4d8d97d5a2a1f2979d41 2024-12-04T19:14:50,726 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/d39ce156f19f4d8d97d5a2a1f2979d41, entries=150, sequenceid=380, filesize=12.0 K 2024-12-04T19:14:50,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/C/2a98a46de8ec430ebc60b7f5b086f1ad as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/2a98a46de8ec430ebc60b7f5b086f1ad 2024-12-04T19:14:50,731 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/2a98a46de8ec430ebc60b7f5b086f1ad, entries=150, sequenceid=380, filesize=12.0 K 2024-12-04T19:14:50,732 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~381.42 KB/390576, currentSize=46.96 KB/48090 for 6fbcb032a86ef6909d8f2378183195f0 in 1685ms, sequenceid=380, compaction requested=true 2024-12-04T19:14:50,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for 6fbcb032a86ef6909d8f2378183195f0: 2024-12-04T19:14:50,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:50,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-12-04T19:14:50,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-12-04T19:14:50,735 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-12-04T19:14:50,735 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8380 sec 2024-12-04T19:14:50,736 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 1.8420 sec 2024-12-04T19:14:50,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(8855): Flush requested on 6fbcb032a86ef6909d8f2378183195f0 2024-12-04T19:14:50,825 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6fbcb032a86ef6909d8f2378183195f0 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-04T19:14:50,825 INFO [Thread-698 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-698. 2024-12-04T19:14:50,825 INFO [Thread-700 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-700. 2024-12-04T19:14:50,825 DEBUG [Thread-698 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T19:14:50,825 DEBUG [Thread-700 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T19:14:50,825 DEBUG [Thread-698 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:14:50,825 DEBUG [Thread-700 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:14:50,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6fbcb032a86ef6909d8f2378183195f0, store=A 2024-12-04T19:14:50,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:14:50,825 DEBUG [Thread-698 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:14:50,825 INFO [Thread-704 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-704. 2024-12-04T19:14:50,825 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T19:14:50,825 DEBUG [Thread-704 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T19:14:50,825 DEBUG [Thread-704 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:14:50,825 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T19:14:50,825 DEBUG [Thread-700 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:14:50,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6fbcb032a86ef6909d8f2378183195f0, store=B 2024-12-04T19:14:50,825 DEBUG [Thread-704 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:14:50,825 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T19:14:50,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:14:50,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6fbcb032a86ef6909d8f2378183195f0, store=C 2024-12-04T19:14:50,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:14:50,834 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412044af35a67d94f4a33924d578f53110587_6fbcb032a86ef6909d8f2378183195f0 is 50, key is test_row_0/A:col10/1733339690823/Put/seqid=0 2024-12-04T19:14:50,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742070_1246 (size=12454) 2024-12-04T19:14:51,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-04T19:14:51,043 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T19:14:51,241 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:51,250 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412044af35a67d94f4a33924d578f53110587_6fbcb032a86ef6909d8f2378183195f0 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412044af35a67d94f4a33924d578f53110587_6fbcb032a86ef6909d8f2378183195f0 2024-12-04T19:14:51,252 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/A/a6afa7a7992642ea9de34bd6cbe689a4, store: [table=TestAcidGuarantees family=A region=6fbcb032a86ef6909d8f2378183195f0] 2024-12-04T19:14:51,253 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/A/a6afa7a7992642ea9de34bd6cbe689a4 is 175, key is test_row_0/A:col10/1733339690823/Put/seqid=0 2024-12-04T19:14:51,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742071_1247 (size=31255) 2024-12-04T19:14:51,660 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=393, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/A/a6afa7a7992642ea9de34bd6cbe689a4 2024-12-04T19:14:51,675 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/B/8e5cab1bc1724798a8a7637f76b49616 is 50, key is test_row_0/B:col10/1733339690823/Put/seqid=0 2024-12-04T19:14:51,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742072_1248 (size=12301) 2024-12-04T19:14:51,704 INFO [Thread-696 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-696. 2024-12-04T19:14:51,704 INFO [Thread-702 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-702. 2024-12-04T19:14:51,704 DEBUG [Thread-696 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T19:14:51,704 DEBUG [Thread-696 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:14:51,704 DEBUG [Thread-702 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T19:14:51,704 DEBUG [Thread-702 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:14:51,704 DEBUG [Thread-696 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:14:51,704 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T19:14:51,704 DEBUG [Thread-702 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:14:51,704 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T19:14:51,704 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-04T19:14:51,705 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 61 2024-12-04T19:14:51,705 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 63 2024-12-04T19:14:51,705 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 61 2024-12-04T19:14:51,705 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 61 2024-12-04T19:14:51,705 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 64 2024-12-04T19:14:51,705 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-04T19:14:51,705 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4475 2024-12-04T19:14:51,705 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4107 2024-12-04T19:14:51,705 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-04T19:14:51,705 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1819 2024-12-04T19:14:51,705 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5456 rows 2024-12-04T19:14:51,705 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1818 2024-12-04T19:14:51,705 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5451 rows 2024-12-04T19:14:51,705 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-04T19:14:51,705 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T19:14:51,705 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:14:51,705 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:14:51,706 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T19:14:51,707 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-04T19:14:51,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-04T19:14:51,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-04T19:14:51,712 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733339691711"}]},"ts":"1733339691711"} 2024-12-04T19:14:51,714 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-04T19:14:51,714 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-04T19:14:51,715 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-04T19:14:51,717 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6fbcb032a86ef6909d8f2378183195f0, UNASSIGN}] 2024-12-04T19:14:51,718 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6fbcb032a86ef6909d8f2378183195f0, UNASSIGN 2024-12-04T19:14:51,719 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=6fbcb032a86ef6909d8f2378183195f0, regionState=CLOSING, regionLocation=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:51,720 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6fbcb032a86ef6909d8f2378183195f0, UNASSIGN because future has completed 2024-12-04T19:14:51,721 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-04T19:14:51,721 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791}] 2024-12-04T19:14:51,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-04T19:14:51,874 INFO [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] handler.UnassignRegionHandler(122): Close 6fbcb032a86ef6909d8f2378183195f0 2024-12-04T19:14:51,875 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-04T19:14:51,875 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1722): Closing 6fbcb032a86ef6909d8f2378183195f0, disabling compactions & flushes 2024-12-04T19:14:51,875 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:52,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-04T19:14:52,081 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=393 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/B/8e5cab1bc1724798a8a7637f76b49616 2024-12-04T19:14:52,097 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/C/9f87fd2c4feb4cf085f65d8acebb67fa is 50, key is test_row_0/C:col10/1733339690823/Put/seqid=0 2024-12-04T19:14:52,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742073_1249 (size=12301) 2024-12-04T19:14:52,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-04T19:14:52,502 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=393 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/C/9f87fd2c4feb4cf085f65d8acebb67fa 2024-12-04T19:14:52,514 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/A/a6afa7a7992642ea9de34bd6cbe689a4 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/a6afa7a7992642ea9de34bd6cbe689a4 2024-12-04T19:14:52,520 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/a6afa7a7992642ea9de34bd6cbe689a4, entries=150, sequenceid=393, filesize=30.5 K 2024-12-04T19:14:52,521 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/B/8e5cab1bc1724798a8a7637f76b49616 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/8e5cab1bc1724798a8a7637f76b49616 2024-12-04T19:14:52,526 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/8e5cab1bc1724798a8a7637f76b49616, entries=150, sequenceid=393, filesize=12.0 K 2024-12-04T19:14:52,526 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/C/9f87fd2c4feb4cf085f65d8acebb67fa as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/9f87fd2c4feb4cf085f65d8acebb67fa 2024-12-04T19:14:52,530 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/9f87fd2c4feb4cf085f65d8acebb67fa, entries=150, sequenceid=393, filesize=12.0 K 2024-12-04T19:14:52,530 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=13.42 KB/13740 for 6fbcb032a86ef6909d8f2378183195f0 in 1706ms, sequenceid=393, compaction requested=true 2024-12-04T19:14:52,531 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6fbcb032a86ef6909d8f2378183195f0: 2024-12-04T19:14:52,531 INFO [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:52,531 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:52,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6fbcb032a86ef6909d8f2378183195f0:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T19:14:52,531 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. after waiting 0 ms 2024-12-04T19:14:52,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:14:52,531 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:52,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6fbcb032a86ef6909d8f2378183195f0:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T19:14:52,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:14:52,531 INFO [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(2902): Flushing 6fbcb032a86ef6909d8f2378183195f0 3/3 column families, dataSize=13.42 KB heapSize=40.59 KB 2024-12-04T19:14:52,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6fbcb032a86ef6909d8f2378183195f0:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T19:14:52,531 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. because compaction request was cancelled 2024-12-04T19:14:52,531 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6fbcb032a86ef6909d8f2378183195f0:A 2024-12-04T19:14:52,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T19:14:52,531 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. because compaction request was cancelled 2024-12-04T19:14:52,531 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. because compaction request was cancelled 2024-12-04T19:14:52,531 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6fbcb032a86ef6909d8f2378183195f0:B 2024-12-04T19:14:52,531 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6fbcb032a86ef6909d8f2378183195f0:C 2024-12-04T19:14:52,531 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6fbcb032a86ef6909d8f2378183195f0, store=A 2024-12-04T19:14:52,531 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:14:52,531 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6fbcb032a86ef6909d8f2378183195f0, store=B 2024-12-04T19:14:52,531 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:14:52,531 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6fbcb032a86ef6909d8f2378183195f0, store=C 2024-12-04T19:14:52,531 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:14:52,536 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120433ecb4e8fcf2426db8fd033facfbf779_6fbcb032a86ef6909d8f2378183195f0 is 50, key is test_row_0/A:col10/1733339691702/Put/seqid=0 2024-12-04T19:14:52,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742074_1250 (size=9914) 2024-12-04T19:14:52,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-04T19:14:52,942 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:52,952 INFO [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120433ecb4e8fcf2426db8fd033facfbf779_6fbcb032a86ef6909d8f2378183195f0 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120433ecb4e8fcf2426db8fd033facfbf779_6fbcb032a86ef6909d8f2378183195f0 2024-12-04T19:14:52,954 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/A/55972ab0a5a840829604d328870a9cb5, store: [table=TestAcidGuarantees family=A region=6fbcb032a86ef6909d8f2378183195f0] 2024-12-04T19:14:52,954 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/A/55972ab0a5a840829604d328870a9cb5 is 175, key is test_row_0/A:col10/1733339691702/Put/seqid=0 2024-12-04T19:14:52,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742075_1251 (size=22561) 2024-12-04T19:14:53,389 INFO [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=398, memsize=4.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/A/55972ab0a5a840829604d328870a9cb5 2024-12-04T19:14:53,404 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/B/8b07f642ce814839836bc7246c0a56a2 is 50, key is test_row_0/B:col10/1733339691702/Put/seqid=0 2024-12-04T19:14:53,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742076_1252 (size=9857) 2024-12-04T19:14:53,810 INFO [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/B/8b07f642ce814839836bc7246c0a56a2 2024-12-04T19:14:53,826 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/C/c21539009c6a4112a6666e2ea7187def is 50, key is test_row_0/C:col10/1733339691702/Put/seqid=0 2024-12-04T19:14:53,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742077_1253 (size=9857) 2024-12-04T19:14:53,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-04T19:14:54,231 INFO [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/C/c21539009c6a4112a6666e2ea7187def 2024-12-04T19:14:54,238 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/A/55972ab0a5a840829604d328870a9cb5 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/55972ab0a5a840829604d328870a9cb5 2024-12-04T19:14:54,243 INFO [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/55972ab0a5a840829604d328870a9cb5, entries=100, sequenceid=398, filesize=22.0 K 2024-12-04T19:14:54,244 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/B/8b07f642ce814839836bc7246c0a56a2 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/8b07f642ce814839836bc7246c0a56a2 2024-12-04T19:14:54,248 INFO [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/8b07f642ce814839836bc7246c0a56a2, entries=100, sequenceid=398, filesize=9.6 K 2024-12-04T19:14:54,249 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/.tmp/C/c21539009c6a4112a6666e2ea7187def as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/c21539009c6a4112a6666e2ea7187def 2024-12-04T19:14:54,254 INFO [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/c21539009c6a4112a6666e2ea7187def, entries=100, sequenceid=398, filesize=9.6 K 2024-12-04T19:14:54,255 INFO [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for 6fbcb032a86ef6909d8f2378183195f0 in 1724ms, sequenceid=398, compaction requested=true 2024-12-04T19:14:54,255 DEBUG [StoreCloser-TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/f1de4ce9a5da4ec5b0f7c61e4d79a706, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/ca0f1cf71d0643ab941daaa49659a425, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/cbc6f06c646b4ecbbea1f4cb572b7b3e, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/24bde5c8899a4fe2abe82b477a05cb48, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/b492be17977f491fa34b1a403da3c778, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/8bcb1e32dab940fab7509fb7597bbfdd, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/081d3d75cfcf4440b0874f81a6c5910b, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/43ab5e2e9ac94c468544b00e4dcb19f1, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/4a51e4252b7a43eeb6f38c925c362222, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/4acbdc9afbf648e393c8596d01b3bcb5, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/f4f82e6b7972423e9b604e329e5f2b18, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/c008e7e326054e5c9a9f05d574a17a58, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/ddf441de1c58454093001a36ada4665a, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/06ab8ea5ec5a4d308fb491f4e2dc3fcb, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/947a0beb7118462e81e5ebc19837dd11, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/10ae7d405d6849ef9f2eb062f40303c1, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/aeae013864ed4f4a99e4109dbefba1ec, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/1bc8ade46f684580bc94cfd1e1149ce6, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/daede8605d454fc6a2857f90e37dca80, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/a9d239a9a46b477e91a74255abb6382d, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/d8358b76c4cc435eac754787a222402e, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/8cbd9fd0d08a4840b3ddc5b7799d45f9, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/8f445a09ee8049a6a746b7837d24cb48] to archive 2024-12-04T19:14:54,256 DEBUG [StoreCloser-TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-04T19:14:54,258 DEBUG [StoreCloser-TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/f1de4ce9a5da4ec5b0f7c61e4d79a706 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/f1de4ce9a5da4ec5b0f7c61e4d79a706 2024-12-04T19:14:54,259 DEBUG [StoreCloser-TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/ca0f1cf71d0643ab941daaa49659a425 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/ca0f1cf71d0643ab941daaa49659a425 2024-12-04T19:14:54,260 DEBUG [StoreCloser-TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/cbc6f06c646b4ecbbea1f4cb572b7b3e to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/cbc6f06c646b4ecbbea1f4cb572b7b3e 2024-12-04T19:14:54,262 DEBUG [StoreCloser-TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/24bde5c8899a4fe2abe82b477a05cb48 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/24bde5c8899a4fe2abe82b477a05cb48 2024-12-04T19:14:54,263 DEBUG [StoreCloser-TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/b492be17977f491fa34b1a403da3c778 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/b492be17977f491fa34b1a403da3c778 2024-12-04T19:14:54,263 DEBUG [StoreCloser-TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/8bcb1e32dab940fab7509fb7597bbfdd to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/8bcb1e32dab940fab7509fb7597bbfdd 2024-12-04T19:14:54,264 DEBUG [StoreCloser-TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/081d3d75cfcf4440b0874f81a6c5910b to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/081d3d75cfcf4440b0874f81a6c5910b 2024-12-04T19:14:54,265 DEBUG [StoreCloser-TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/43ab5e2e9ac94c468544b00e4dcb19f1 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/43ab5e2e9ac94c468544b00e4dcb19f1 2024-12-04T19:14:54,266 DEBUG [StoreCloser-TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/4a51e4252b7a43eeb6f38c925c362222 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/4a51e4252b7a43eeb6f38c925c362222 2024-12-04T19:14:54,267 DEBUG [StoreCloser-TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/4acbdc9afbf648e393c8596d01b3bcb5 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/4acbdc9afbf648e393c8596d01b3bcb5 2024-12-04T19:14:54,268 DEBUG [StoreCloser-TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/f4f82e6b7972423e9b604e329e5f2b18 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/f4f82e6b7972423e9b604e329e5f2b18 2024-12-04T19:14:54,269 DEBUG [StoreCloser-TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/c008e7e326054e5c9a9f05d574a17a58 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/c008e7e326054e5c9a9f05d574a17a58 2024-12-04T19:14:54,269 DEBUG [StoreCloser-TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/ddf441de1c58454093001a36ada4665a to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/ddf441de1c58454093001a36ada4665a 2024-12-04T19:14:54,270 DEBUG [StoreCloser-TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/06ab8ea5ec5a4d308fb491f4e2dc3fcb to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/06ab8ea5ec5a4d308fb491f4e2dc3fcb 2024-12-04T19:14:54,271 DEBUG [StoreCloser-TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/947a0beb7118462e81e5ebc19837dd11 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/947a0beb7118462e81e5ebc19837dd11 2024-12-04T19:14:54,272 DEBUG [StoreCloser-TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/10ae7d405d6849ef9f2eb062f40303c1 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/10ae7d405d6849ef9f2eb062f40303c1 2024-12-04T19:14:54,273 DEBUG [StoreCloser-TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/aeae013864ed4f4a99e4109dbefba1ec to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/aeae013864ed4f4a99e4109dbefba1ec 2024-12-04T19:14:54,274 DEBUG [StoreCloser-TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/1bc8ade46f684580bc94cfd1e1149ce6 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/1bc8ade46f684580bc94cfd1e1149ce6 2024-12-04T19:14:54,275 DEBUG [StoreCloser-TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/daede8605d454fc6a2857f90e37dca80 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/daede8605d454fc6a2857f90e37dca80 2024-12-04T19:14:54,275 DEBUG [StoreCloser-TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/a9d239a9a46b477e91a74255abb6382d to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/a9d239a9a46b477e91a74255abb6382d 2024-12-04T19:14:54,276 DEBUG [StoreCloser-TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/d8358b76c4cc435eac754787a222402e to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/d8358b76c4cc435eac754787a222402e 2024-12-04T19:14:54,277 DEBUG [StoreCloser-TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/8cbd9fd0d08a4840b3ddc5b7799d45f9 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/8cbd9fd0d08a4840b3ddc5b7799d45f9 2024-12-04T19:14:54,278 DEBUG [StoreCloser-TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/8f445a09ee8049a6a746b7837d24cb48 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/8f445a09ee8049a6a746b7837d24cb48 2024-12-04T19:14:54,279 DEBUG [StoreCloser-TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/e83d0ddf343c45fca0441609aff38cf5, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/551081890a48437a94cb83422ae43980, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/8a9139e5337b41a1bbb61369ff7238bb, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/d69d333592314c30aac7279c4748c5c9, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/acdaa6f2fa304f5982d055ad6202187f, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/d47a7f08b5674ae9acbdd428e6689c5c, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/38f2c7f850f345789efcde7eab8062fd, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/621e27b2f2824f13940958479d95705c, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/891deabd92bb4d6c9d9d318abb75557f, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/bbce90ea7b044c13856aa17fda134dd8, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/9ac9c5e06a1f40ff8f93ea724e2cdaf4, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/610fef7ddfea402891bb5cd70796136b, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/7eec59502ecd4134a1fd60367199d946, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/aa7ee4925ba64e3ebcab1681c4b5451f, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/4f2429a4e66e4ae782dbfaa92de9eabd, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/e16c5b785a4b470d89e7f8f1a69c527b, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/aa98a8d251a147b58f93329df905d73a, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/98a662f73f984b858a5df8d91b32a34e, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/c042a5ae2a424c57bbb5ecf7b2170df1, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/097815f341504b389f5f9769111ccc21, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/ec711d999c2f4799b48ca45a48599ce2, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/14691e53cb7c43d2a6cd2752c74c1666, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/acff263634a74f949bf45a8de0ba0937] to archive 2024-12-04T19:14:54,280 DEBUG [StoreCloser-TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-04T19:14:54,281 DEBUG [StoreCloser-TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/e83d0ddf343c45fca0441609aff38cf5 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/e83d0ddf343c45fca0441609aff38cf5 2024-12-04T19:14:54,282 DEBUG [StoreCloser-TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/551081890a48437a94cb83422ae43980 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/551081890a48437a94cb83422ae43980 2024-12-04T19:14:54,284 DEBUG [StoreCloser-TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/8a9139e5337b41a1bbb61369ff7238bb to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/8a9139e5337b41a1bbb61369ff7238bb 2024-12-04T19:14:54,284 DEBUG [StoreCloser-TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/d69d333592314c30aac7279c4748c5c9 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/d69d333592314c30aac7279c4748c5c9 2024-12-04T19:14:54,285 DEBUG [StoreCloser-TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/acdaa6f2fa304f5982d055ad6202187f to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/acdaa6f2fa304f5982d055ad6202187f 2024-12-04T19:14:54,286 DEBUG [StoreCloser-TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/d47a7f08b5674ae9acbdd428e6689c5c to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/d47a7f08b5674ae9acbdd428e6689c5c 2024-12-04T19:14:54,287 DEBUG [StoreCloser-TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/38f2c7f850f345789efcde7eab8062fd to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/38f2c7f850f345789efcde7eab8062fd 2024-12-04T19:14:54,288 DEBUG [StoreCloser-TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/621e27b2f2824f13940958479d95705c to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/621e27b2f2824f13940958479d95705c 2024-12-04T19:14:54,289 DEBUG [StoreCloser-TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/891deabd92bb4d6c9d9d318abb75557f to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/891deabd92bb4d6c9d9d318abb75557f 2024-12-04T19:14:54,290 DEBUG [StoreCloser-TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/bbce90ea7b044c13856aa17fda134dd8 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/bbce90ea7b044c13856aa17fda134dd8 2024-12-04T19:14:54,291 DEBUG [StoreCloser-TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/9ac9c5e06a1f40ff8f93ea724e2cdaf4 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/9ac9c5e06a1f40ff8f93ea724e2cdaf4 2024-12-04T19:14:54,292 DEBUG [StoreCloser-TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/610fef7ddfea402891bb5cd70796136b to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/610fef7ddfea402891bb5cd70796136b 2024-12-04T19:14:54,293 DEBUG [StoreCloser-TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/7eec59502ecd4134a1fd60367199d946 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/7eec59502ecd4134a1fd60367199d946 2024-12-04T19:14:54,294 DEBUG [StoreCloser-TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/aa7ee4925ba64e3ebcab1681c4b5451f to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/aa7ee4925ba64e3ebcab1681c4b5451f 2024-12-04T19:14:54,295 DEBUG [StoreCloser-TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/4f2429a4e66e4ae782dbfaa92de9eabd to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/4f2429a4e66e4ae782dbfaa92de9eabd 2024-12-04T19:14:54,295 DEBUG [StoreCloser-TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/e16c5b785a4b470d89e7f8f1a69c527b to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/e16c5b785a4b470d89e7f8f1a69c527b 2024-12-04T19:14:54,296 DEBUG [StoreCloser-TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/aa98a8d251a147b58f93329df905d73a to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/aa98a8d251a147b58f93329df905d73a 2024-12-04T19:14:54,297 DEBUG [StoreCloser-TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/98a662f73f984b858a5df8d91b32a34e to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/98a662f73f984b858a5df8d91b32a34e 2024-12-04T19:14:54,298 DEBUG [StoreCloser-TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/c042a5ae2a424c57bbb5ecf7b2170df1 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/c042a5ae2a424c57bbb5ecf7b2170df1 2024-12-04T19:14:54,299 DEBUG [StoreCloser-TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/097815f341504b389f5f9769111ccc21 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/097815f341504b389f5f9769111ccc21 2024-12-04T19:14:54,300 DEBUG [StoreCloser-TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/ec711d999c2f4799b48ca45a48599ce2 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/ec711d999c2f4799b48ca45a48599ce2 2024-12-04T19:14:54,300 DEBUG [StoreCloser-TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/14691e53cb7c43d2a6cd2752c74c1666 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/14691e53cb7c43d2a6cd2752c74c1666 2024-12-04T19:14:54,301 DEBUG [StoreCloser-TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/acff263634a74f949bf45a8de0ba0937 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/acff263634a74f949bf45a8de0ba0937 2024-12-04T19:14:54,302 DEBUG [StoreCloser-TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/e51ba5f5037348e4afce3ad466fae247, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/a4ae14e5e2714d06aa76e3baec83b50b, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/25355aa88e5c44929671b0407beb3d1b, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/ebd2c8f6fed543c6b29eb7b714e0847e, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/d3ecfd4e2363449cb0dcc76ec1b4fd2f, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/127107817f34420d91cba0daae31db6c, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/d39e2d243fdc460aba6c6ce2e8682e11, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/74b40f3fff454313afa5b77040c52225, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/38e52066eb70483d95af7ee12f286fa3, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/64e269733fe541ebb69d47cb22fbacc4, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/588ef0f1f7fa4759b9ed7ea9b5f52fcf, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/ead88345670e4b1aa1cf2a091a656399, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/23421e154e064cf6860ce386f6c86dc2, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/1f6865ca0c9d4c3b8574577aab7af30e, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/d2911bdede90402c9b9f3f019ec01f33, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/7405d71906e0410b88e76248bb6eaf71, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/960d4db894c14ec0b77a4a3604c1208c, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/c9d22bffd94b4206ba50ccbda3b04aa7, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/0d9dec93cec147b58564f84105e15252, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/e8e07769cd32443ab3f4e000d227a0f4, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/7c57cb73f6734e568c0d8bbc673b1d14, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/d616df55eaeb4e18b0d1d55c3983129d, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/72ae60f0031143caa84748b3ef99af38] to archive 2024-12-04T19:14:54,303 DEBUG [StoreCloser-TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-04T19:14:54,304 DEBUG [StoreCloser-TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/e51ba5f5037348e4afce3ad466fae247 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/e51ba5f5037348e4afce3ad466fae247 2024-12-04T19:14:54,305 DEBUG [StoreCloser-TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/a4ae14e5e2714d06aa76e3baec83b50b to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/a4ae14e5e2714d06aa76e3baec83b50b 2024-12-04T19:14:54,306 DEBUG [StoreCloser-TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/25355aa88e5c44929671b0407beb3d1b to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/25355aa88e5c44929671b0407beb3d1b 2024-12-04T19:14:54,306 DEBUG [StoreCloser-TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/ebd2c8f6fed543c6b29eb7b714e0847e to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/ebd2c8f6fed543c6b29eb7b714e0847e 2024-12-04T19:14:54,307 DEBUG [StoreCloser-TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/d3ecfd4e2363449cb0dcc76ec1b4fd2f to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/d3ecfd4e2363449cb0dcc76ec1b4fd2f 2024-12-04T19:14:54,308 DEBUG [StoreCloser-TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/127107817f34420d91cba0daae31db6c to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/127107817f34420d91cba0daae31db6c 2024-12-04T19:14:54,308 DEBUG [StoreCloser-TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/d39e2d243fdc460aba6c6ce2e8682e11 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/d39e2d243fdc460aba6c6ce2e8682e11 2024-12-04T19:14:54,309 DEBUG [StoreCloser-TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/74b40f3fff454313afa5b77040c52225 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/74b40f3fff454313afa5b77040c52225 2024-12-04T19:14:54,310 DEBUG [StoreCloser-TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/38e52066eb70483d95af7ee12f286fa3 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/38e52066eb70483d95af7ee12f286fa3 2024-12-04T19:14:54,310 DEBUG [StoreCloser-TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/64e269733fe541ebb69d47cb22fbacc4 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/64e269733fe541ebb69d47cb22fbacc4 2024-12-04T19:14:54,311 DEBUG [StoreCloser-TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/588ef0f1f7fa4759b9ed7ea9b5f52fcf to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/588ef0f1f7fa4759b9ed7ea9b5f52fcf 2024-12-04T19:14:54,312 DEBUG [StoreCloser-TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/ead88345670e4b1aa1cf2a091a656399 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/ead88345670e4b1aa1cf2a091a656399 2024-12-04T19:14:54,313 DEBUG [StoreCloser-TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/23421e154e064cf6860ce386f6c86dc2 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/23421e154e064cf6860ce386f6c86dc2 2024-12-04T19:14:54,313 DEBUG [StoreCloser-TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/1f6865ca0c9d4c3b8574577aab7af30e to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/1f6865ca0c9d4c3b8574577aab7af30e 2024-12-04T19:14:54,314 DEBUG [StoreCloser-TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/d2911bdede90402c9b9f3f019ec01f33 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/d2911bdede90402c9b9f3f019ec01f33 2024-12-04T19:14:54,314 DEBUG [StoreCloser-TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/7405d71906e0410b88e76248bb6eaf71 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/7405d71906e0410b88e76248bb6eaf71 2024-12-04T19:14:54,315 DEBUG [StoreCloser-TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/960d4db894c14ec0b77a4a3604c1208c to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/960d4db894c14ec0b77a4a3604c1208c 2024-12-04T19:14:54,316 DEBUG [StoreCloser-TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/c9d22bffd94b4206ba50ccbda3b04aa7 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/c9d22bffd94b4206ba50ccbda3b04aa7 2024-12-04T19:14:54,317 DEBUG [StoreCloser-TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/0d9dec93cec147b58564f84105e15252 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/0d9dec93cec147b58564f84105e15252 2024-12-04T19:14:54,318 DEBUG [StoreCloser-TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/e8e07769cd32443ab3f4e000d227a0f4 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/e8e07769cd32443ab3f4e000d227a0f4 2024-12-04T19:14:54,319 DEBUG [StoreCloser-TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/7c57cb73f6734e568c0d8bbc673b1d14 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/7c57cb73f6734e568c0d8bbc673b1d14 2024-12-04T19:14:54,320 DEBUG [StoreCloser-TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/d616df55eaeb4e18b0d1d55c3983129d to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/d616df55eaeb4e18b0d1d55c3983129d 2024-12-04T19:14:54,321 DEBUG [StoreCloser-TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/72ae60f0031143caa84748b3ef99af38 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/72ae60f0031143caa84748b3ef99af38 2024-12-04T19:14:54,325 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/recovered.edits/401.seqid, newMaxSeqId=401, maxSeqId=4 2024-12-04T19:14:54,326 INFO [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. 2024-12-04T19:14:54,326 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1676): Region close journal for 6fbcb032a86ef6909d8f2378183195f0: Waiting for close lock at 1733339691875Running coprocessor pre-close hooks at 1733339691875Disabling compacts and flushes for region at 1733339691875Disabling writes for close at 1733339692531 (+656 ms)Obtaining lock to block concurrent updates at 1733339692531Preparing flush snapshotting stores in 6fbcb032a86ef6909d8f2378183195f0 at 1733339692531Finished memstore snapshotting TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=41520, getOffHeapSize=0, getCellsCount=300 at 1733339692531Flushing stores of TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0. at 1733339692531Flushing 6fbcb032a86ef6909d8f2378183195f0/A: creating writer at 1733339692531Flushing mob file 6fbcb032a86ef6909d8f2378183195f0/A: appending metadata at 1733339692536 (+5 ms)Flushing mob file 6fbcb032a86ef6909d8f2378183195f0/A: closing flushed file at 1733339692536Flushing 6fbcb032a86ef6909d8f2378183195f0/A: appending metadata at 1733339692954 (+418 ms)Flushing 6fbcb032a86ef6909d8f2378183195f0/A: closing flushed file at 1733339692954Flushing 6fbcb032a86ef6909d8f2378183195f0/B: creating writer at 1733339693400 (+446 ms)Flushing 6fbcb032a86ef6909d8f2378183195f0/B: appending metadata at 1733339693403 (+3 ms)Flushing 6fbcb032a86ef6909d8f2378183195f0/B: closing flushed file at 1733339693403Flushing 6fbcb032a86ef6909d8f2378183195f0/C: creating writer at 1733339693819 (+416 ms)Flushing 6fbcb032a86ef6909d8f2378183195f0/C: appending metadata at 1733339693825 (+6 ms)Flushing 6fbcb032a86ef6909d8f2378183195f0/C: closing flushed file at 1733339693825Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@12e8ec0b: reopening flushed file at 1733339694237 (+412 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@56130e49: reopening flushed file at 1733339694243 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@40d03e3b: reopening flushed file at 1733339694248 (+5 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for 6fbcb032a86ef6909d8f2378183195f0 in 1724ms, sequenceid=398, compaction requested=true at 1733339694255 (+7 ms)Writing region close event to WAL at 1733339694322 (+67 ms)Running coprocessor post-close hooks at 1733339694325 (+3 ms)Closed at 1733339694326 (+1 ms) 2024-12-04T19:14:54,328 INFO [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] handler.UnassignRegionHandler(157): Closed 6fbcb032a86ef6909d8f2378183195f0 2024-12-04T19:14:54,329 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=6fbcb032a86ef6909d8f2378183195f0, regionState=CLOSED 2024-12-04T19:14:54,330 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=59, ppid=58, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 because future has completed 2024-12-04T19:14:54,332 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-12-04T19:14:54,333 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; CloseRegionProcedure 6fbcb032a86ef6909d8f2378183195f0, server=9ba2734cd39f,33969,1733339629791 in 2.6100 sec 2024-12-04T19:14:54,334 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-12-04T19:14:54,334 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6fbcb032a86ef6909d8f2378183195f0, UNASSIGN in 2.6160 sec 2024-12-04T19:14:54,336 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-12-04T19:14:54,336 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.6200 sec 2024-12-04T19:14:54,337 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733339694337"}]},"ts":"1733339694337"} 2024-12-04T19:14:54,339 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-04T19:14:54,339 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-04T19:14:54,340 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.6320 sec 2024-12-04T19:14:55,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-04T19:14:55,873 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-04T19:14:55,874 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-04T19:14:55,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T19:14:55,877 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=60, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T19:14:55,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-04T19:14:55,878 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=60, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T19:14:55,881 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0 2024-12-04T19:14:55,884 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A, FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B, FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C, FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/recovered.edits] 2024-12-04T19:14:55,886 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/0813e33b19d14c3d94bc5ae8a4f5e476 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/0813e33b19d14c3d94bc5ae8a4f5e476 2024-12-04T19:14:55,887 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/55972ab0a5a840829604d328870a9cb5 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/55972ab0a5a840829604d328870a9cb5 2024-12-04T19:14:55,888 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/95c2cd46f96149e2af0606b75f313325 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/95c2cd46f96149e2af0606b75f313325 2024-12-04T19:14:55,889 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/a6afa7a7992642ea9de34bd6cbe689a4 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/a6afa7a7992642ea9de34bd6cbe689a4 2024-12-04T19:14:55,890 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/f02e6be1b5b74b549fcda387394a9f7c to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/A/f02e6be1b5b74b549fcda387394a9f7c 2024-12-04T19:14:55,892 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/8b07f642ce814839836bc7246c0a56a2 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/8b07f642ce814839836bc7246c0a56a2 2024-12-04T19:14:55,892 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/8e5cab1bc1724798a8a7637f76b49616 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/8e5cab1bc1724798a8a7637f76b49616 2024-12-04T19:14:55,893 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/d39ce156f19f4d8d97d5a2a1f2979d41 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/d39ce156f19f4d8d97d5a2a1f2979d41 2024-12-04T19:14:55,894 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/d984aac7c9e14133a920c6878e0b981b to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/d984aac7c9e14133a920c6878e0b981b 2024-12-04T19:14:55,895 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/efe980b4c3e34779bad7af9823d9e368 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/B/efe980b4c3e34779bad7af9823d9e368 2024-12-04T19:14:55,896 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/00f847659b1548f2b74ad06a26d41a99 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/00f847659b1548f2b74ad06a26d41a99 2024-12-04T19:14:55,897 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/2a98a46de8ec430ebc60b7f5b086f1ad to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/2a98a46de8ec430ebc60b7f5b086f1ad 2024-12-04T19:14:55,898 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/4e6383bf26ad4bd6b3a58ceb9d4b908d to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/4e6383bf26ad4bd6b3a58ceb9d4b908d 2024-12-04T19:14:55,899 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/9f87fd2c4feb4cf085f65d8acebb67fa to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/9f87fd2c4feb4cf085f65d8acebb67fa 2024-12-04T19:14:55,900 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/c21539009c6a4112a6666e2ea7187def to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/C/c21539009c6a4112a6666e2ea7187def 2024-12-04T19:14:55,902 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/recovered.edits/401.seqid to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0/recovered.edits/401.seqid 2024-12-04T19:14:55,902 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6fbcb032a86ef6909d8f2378183195f0 2024-12-04T19:14:55,902 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-04T19:14:55,903 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-04T19:14:55,903 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-04T19:14:55,906 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204000df985678347b3bf85c43ffa4b4327_6fbcb032a86ef6909d8f2378183195f0 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204000df985678347b3bf85c43ffa4b4327_6fbcb032a86ef6909d8f2378183195f0 2024-12-04T19:14:55,907 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120407e5cf1637fc4771a0e2a6c37d528d13_6fbcb032a86ef6909d8f2378183195f0 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120407e5cf1637fc4771a0e2a6c37d528d13_6fbcb032a86ef6909d8f2378183195f0 2024-12-04T19:14:55,908 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120407f241551bf3424b89da4f683ecfddda_6fbcb032a86ef6909d8f2378183195f0 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120407f241551bf3424b89da4f683ecfddda_6fbcb032a86ef6909d8f2378183195f0 2024-12-04T19:14:55,909 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412040b13366b55b2412b9fbcf388cfee31bf_6fbcb032a86ef6909d8f2378183195f0 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412040b13366b55b2412b9fbcf388cfee31bf_6fbcb032a86ef6909d8f2378183195f0 2024-12-04T19:14:55,910 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412040f3c2ea3f091409ab9fb25fc9831adaf_6fbcb032a86ef6909d8f2378183195f0 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412040f3c2ea3f091409ab9fb25fc9831adaf_6fbcb032a86ef6909d8f2378183195f0 2024-12-04T19:14:55,911 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204140f0642e2d2442aab37e0b87fb4cc1e_6fbcb032a86ef6909d8f2378183195f0 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204140f0642e2d2442aab37e0b87fb4cc1e_6fbcb032a86ef6909d8f2378183195f0 2024-12-04T19:14:55,912 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120433ecb4e8fcf2426db8fd033facfbf779_6fbcb032a86ef6909d8f2378183195f0 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120433ecb4e8fcf2426db8fd033facfbf779_6fbcb032a86ef6909d8f2378183195f0 2024-12-04T19:14:55,913 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120449fb09bdcddc45afab85361cd7c3b5a8_6fbcb032a86ef6909d8f2378183195f0 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120449fb09bdcddc45afab85361cd7c3b5a8_6fbcb032a86ef6909d8f2378183195f0 2024-12-04T19:14:55,913 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412044a6dd891a98543a0a6861ae6f18e6e9e_6fbcb032a86ef6909d8f2378183195f0 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412044a6dd891a98543a0a6861ae6f18e6e9e_6fbcb032a86ef6909d8f2378183195f0 2024-12-04T19:14:55,914 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412044af35a67d94f4a33924d578f53110587_6fbcb032a86ef6909d8f2378183195f0 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412044af35a67d94f4a33924d578f53110587_6fbcb032a86ef6909d8f2378183195f0 2024-12-04T19:14:55,915 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412045039d25f09a54b9fac07f952a177ae91_6fbcb032a86ef6909d8f2378183195f0 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412045039d25f09a54b9fac07f952a177ae91_6fbcb032a86ef6909d8f2378183195f0 2024-12-04T19:14:55,916 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412045c31fdacdfa24346b9f16bb7845d40ae_6fbcb032a86ef6909d8f2378183195f0 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412045c31fdacdfa24346b9f16bb7845d40ae_6fbcb032a86ef6909d8f2378183195f0 2024-12-04T19:14:55,917 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204989cd70cec92495289017730a671d0b7_6fbcb032a86ef6909d8f2378183195f0 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204989cd70cec92495289017730a671d0b7_6fbcb032a86ef6909d8f2378183195f0 2024-12-04T19:14:55,917 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204aa02a948c3b547af9d2c3cbc3b2fa163_6fbcb032a86ef6909d8f2378183195f0 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204aa02a948c3b547af9d2c3cbc3b2fa163_6fbcb032a86ef6909d8f2378183195f0 2024-12-04T19:14:55,918 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204b914aa7d61fd4811bb9b59185929fed9_6fbcb032a86ef6909d8f2378183195f0 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204b914aa7d61fd4811bb9b59185929fed9_6fbcb032a86ef6909d8f2378183195f0 2024-12-04T19:14:55,919 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204da4871ac2f0242fbafa194b094af5aaf_6fbcb032a86ef6909d8f2378183195f0 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204da4871ac2f0242fbafa194b094af5aaf_6fbcb032a86ef6909d8f2378183195f0 2024-12-04T19:14:55,920 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204ddc662beb8224a169953fec38deaddba_6fbcb032a86ef6909d8f2378183195f0 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204ddc662beb8224a169953fec38deaddba_6fbcb032a86ef6909d8f2378183195f0 2024-12-04T19:14:55,921 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204df46ed2f552546ada170041ede8c9633_6fbcb032a86ef6909d8f2378183195f0 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204df46ed2f552546ada170041ede8c9633_6fbcb032a86ef6909d8f2378183195f0 2024-12-04T19:14:55,921 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204e34d6008a0c240ae8e719c8f44ddf206_6fbcb032a86ef6909d8f2378183195f0 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204e34d6008a0c240ae8e719c8f44ddf206_6fbcb032a86ef6909d8f2378183195f0 2024-12-04T19:14:55,922 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204e546894f9b80495c95622f5058babb50_6fbcb032a86ef6909d8f2378183195f0 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204e546894f9b80495c95622f5058babb50_6fbcb032a86ef6909d8f2378183195f0 2024-12-04T19:14:55,923 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204f1ecb06b0f474160801a1f1a83cf6ded_6fbcb032a86ef6909d8f2378183195f0 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204f1ecb06b0f474160801a1f1a83cf6ded_6fbcb032a86ef6909d8f2378183195f0 2024-12-04T19:14:55,924 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-04T19:14:55,925 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=60, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T19:14:55,928 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-04T19:14:55,930 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-04T19:14:55,931 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=60, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T19:14:55,931 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-04T19:14:55,931 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733339695931"}]},"ts":"9223372036854775807"} 2024-12-04T19:14:55,934 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-04T19:14:55,934 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 6fbcb032a86ef6909d8f2378183195f0, NAME => 'TestAcidGuarantees,,1733339667005.6fbcb032a86ef6909d8f2378183195f0.', STARTKEY => '', ENDKEY => ''}] 2024-12-04T19:14:55,935 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-04T19:14:55,935 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733339695935"}]},"ts":"9223372036854775807"} 2024-12-04T19:14:55,937 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-04T19:14:55,938 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=60, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T19:14:55,940 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 65 msec 2024-12-04T19:14:55,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-04T19:14:55,982 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-04T19:14:55,982 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-04T19:14:55,991 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=252 (was 247) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_565853306_22 at /127.0.0.1:41852 [Waiting for operation #699] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1db4ba39-9940-6e0b-9624-ffcf7fe27afc/cluster_45aec480-86fd-40c3-b6dd-2ba84309c8cd/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_565853306_22 at /127.0.0.1:41752 [Waiting for operation #708] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1487874761_22 at /127.0.0.1:41658 [Waiting for operation #478] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1db4ba39-9940-6e0b-9624-ffcf7fe27afc/cluster_45aec480-86fd-40c3-b6dd-2ba84309c8cd/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1487874761_22 at /127.0.0.1:41662 [Waiting for operation #477] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=459 (was 449) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=429 (was 361) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=15034 (was 15205) 2024-12-04T19:14:55,999 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=252, OpenFileDescriptor=459, MaxFileDescriptor=1048576, SystemLoadAverage=429, ProcessCount=11, AvailableMemoryMB=15034 2024-12-04T19:14:56,000 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-04T19:14:56,000 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T19:14:56,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] procedure2.ProcedureExecutor(1139): Stored pid=61, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-04T19:14:56,002 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-04T19:14:56,002 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:56,002 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 61 2024-12-04T19:14:56,003 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-04T19:14:56,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-12-04T19:14:56,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742078_1254 (size=1061) 2024-12-04T19:14:56,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-12-04T19:14:56,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-12-04T19:14:56,415 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 6c5c1603a070441ac38abd9294c86a68, NAME => 'TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d 2024-12-04T19:14:56,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742079_1255 (size=53) 2024-12-04T19:14:56,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-12-04T19:14:56,826 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T19:14:56,827 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 6c5c1603a070441ac38abd9294c86a68, disabling compactions & flushes 2024-12-04T19:14:56,827 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. 2024-12-04T19:14:56,827 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. 2024-12-04T19:14:56,827 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. after waiting 0 ms 2024-12-04T19:14:56,827 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. 2024-12-04T19:14:56,827 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. 2024-12-04T19:14:56,827 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 6c5c1603a070441ac38abd9294c86a68: Waiting for close lock at 1733339696827Disabling compacts and flushes for region at 1733339696827Disabling writes for close at 1733339696827Writing region close event to WAL at 1733339696827Closed at 1733339696827 2024-12-04T19:14:56,831 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-04T19:14:56,831 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733339696831"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733339696831"}]},"ts":"1733339696831"} 2024-12-04T19:14:56,833 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-04T19:14:56,834 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-04T19:14:56,834 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733339696834"}]},"ts":"1733339696834"} 2024-12-04T19:14:56,836 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-04T19:14:56,836 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6c5c1603a070441ac38abd9294c86a68, ASSIGN}] 2024-12-04T19:14:56,837 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6c5c1603a070441ac38abd9294c86a68, ASSIGN 2024-12-04T19:14:56,838 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6c5c1603a070441ac38abd9294c86a68, ASSIGN; state=OFFLINE, location=9ba2734cd39f,33969,1733339629791; forceNewPlan=false, retain=false 2024-12-04T19:14:56,989 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=6c5c1603a070441ac38abd9294c86a68, regionState=OPENING, regionLocation=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:56,994 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6c5c1603a070441ac38abd9294c86a68, ASSIGN because future has completed 2024-12-04T19:14:56,995 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791}] 2024-12-04T19:14:57,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-12-04T19:14:57,155 INFO [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. 2024-12-04T19:14:57,155 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7752): Opening region: {ENCODED => 6c5c1603a070441ac38abd9294c86a68, NAME => 'TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.', STARTKEY => '', ENDKEY => ''} 2024-12-04T19:14:57,156 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 6c5c1603a070441ac38abd9294c86a68 2024-12-04T19:14:57,156 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T19:14:57,156 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7794): checking encryption for 6c5c1603a070441ac38abd9294c86a68 2024-12-04T19:14:57,156 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7797): checking classloading for 6c5c1603a070441ac38abd9294c86a68 2024-12-04T19:14:57,160 INFO [StoreOpener-6c5c1603a070441ac38abd9294c86a68-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 6c5c1603a070441ac38abd9294c86a68 2024-12-04T19:14:57,162 INFO [StoreOpener-6c5c1603a070441ac38abd9294c86a68-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-04T19:14:57,163 INFO [StoreOpener-6c5c1603a070441ac38abd9294c86a68-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6c5c1603a070441ac38abd9294c86a68 columnFamilyName A 2024-12-04T19:14:57,163 DEBUG [StoreOpener-6c5c1603a070441ac38abd9294c86a68-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:57,164 INFO [StoreOpener-6c5c1603a070441ac38abd9294c86a68-1 {}] regionserver.HStore(327): Store=6c5c1603a070441ac38abd9294c86a68/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:14:57,164 INFO [StoreOpener-6c5c1603a070441ac38abd9294c86a68-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 6c5c1603a070441ac38abd9294c86a68 2024-12-04T19:14:57,166 INFO [StoreOpener-6c5c1603a070441ac38abd9294c86a68-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-04T19:14:57,166 INFO [StoreOpener-6c5c1603a070441ac38abd9294c86a68-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6c5c1603a070441ac38abd9294c86a68 columnFamilyName B 2024-12-04T19:14:57,166 DEBUG [StoreOpener-6c5c1603a070441ac38abd9294c86a68-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:57,166 INFO [StoreOpener-6c5c1603a070441ac38abd9294c86a68-1 {}] regionserver.HStore(327): Store=6c5c1603a070441ac38abd9294c86a68/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:14:57,167 INFO [StoreOpener-6c5c1603a070441ac38abd9294c86a68-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 6c5c1603a070441ac38abd9294c86a68 2024-12-04T19:14:57,168 INFO [StoreOpener-6c5c1603a070441ac38abd9294c86a68-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-04T19:14:57,168 INFO [StoreOpener-6c5c1603a070441ac38abd9294c86a68-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6c5c1603a070441ac38abd9294c86a68 columnFamilyName C 2024-12-04T19:14:57,168 DEBUG [StoreOpener-6c5c1603a070441ac38abd9294c86a68-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:14:57,169 INFO [StoreOpener-6c5c1603a070441ac38abd9294c86a68-1 {}] regionserver.HStore(327): Store=6c5c1603a070441ac38abd9294c86a68/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:14:57,169 INFO [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. 2024-12-04T19:14:57,169 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1038): replaying wal for 6c5c1603a070441ac38abd9294c86a68 2024-12-04T19:14:57,170 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68 2024-12-04T19:14:57,170 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68 2024-12-04T19:14:57,172 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1048): stopping wal replay for 6c5c1603a070441ac38abd9294c86a68 2024-12-04T19:14:57,172 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1060): Cleaning up temporary data for 6c5c1603a070441ac38abd9294c86a68 2024-12-04T19:14:57,172 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-04T19:14:57,174 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1093): writing seq id for 6c5c1603a070441ac38abd9294c86a68 2024-12-04T19:14:57,176 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T19:14:57,176 INFO [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1114): Opened 6c5c1603a070441ac38abd9294c86a68; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75429776, jitterRate=0.12399125099182129}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-04T19:14:57,176 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6c5c1603a070441ac38abd9294c86a68 2024-12-04T19:14:57,177 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1006): Region open journal for 6c5c1603a070441ac38abd9294c86a68: Running coprocessor pre-open hook at 1733339697157Writing region info on filesystem at 1733339697157Initializing all the Stores at 1733339697159 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733339697159Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733339697159Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733339697159Cleaning up temporary data from old regions at 1733339697172 (+13 ms)Running coprocessor post-open hooks at 1733339697176 (+4 ms)Region opened successfully at 1733339697177 (+1 ms) 2024-12-04T19:14:57,178 INFO [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., pid=63, masterSystemTime=1733339697148 2024-12-04T19:14:57,180 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. 2024-12-04T19:14:57,180 INFO [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. 2024-12-04T19:14:57,180 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=6c5c1603a070441ac38abd9294c86a68, regionState=OPEN, openSeqNum=2, regionLocation=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:57,182 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=63, ppid=62, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 because future has completed 2024-12-04T19:14:57,185 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-12-04T19:14:57,185 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; OpenRegionProcedure 6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 in 188 msec 2024-12-04T19:14:57,188 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=61 2024-12-04T19:14:57,188 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=61, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6c5c1603a070441ac38abd9294c86a68, ASSIGN in 349 msec 2024-12-04T19:14:57,188 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-04T19:14:57,188 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733339697188"}]},"ts":"1733339697188"} 2024-12-04T19:14:57,190 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-04T19:14:57,190 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-04T19:14:57,191 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1900 sec 2024-12-04T19:14:58,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-12-04T19:14:58,152 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-04T19:14:58,153 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@127229bf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:14:58,153 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ba2734cd39f,46667,-1 for getting cluster id 2024-12-04T19:14:58,154 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T19:14:58,155 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9baaea35-8dd3-4653-9b7c-03e5b792e644' 2024-12-04T19:14:58,155 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T19:14:58,155 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9baaea35-8dd3-4653-9b7c-03e5b792e644" 2024-12-04T19:14:58,155 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@776e9311, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:14:58,155 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ba2734cd39f,46667,-1] 2024-12-04T19:14:58,155 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T19:14:58,156 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:14:58,157 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40456, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T19:14:58,157 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4529dcd3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:14:58,157 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T19:14:58,158 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ba2734cd39f,33969,1733339629791, seqNum=-1] 2024-12-04T19:14:58,159 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T19:14:58,160 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37170, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T19:14:58,161 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-04T19:14:58,162 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 9ba2734cd39f,46667,1733339628877 2024-12-04T19:14:58,162 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1674d0fa 2024-12-04T19:14:58,162 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-04T19:14:58,164 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40468, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-04T19:14:58,166 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28324889, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:14:58,166 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ba2734cd39f,46667,-1 for getting cluster id 2024-12-04T19:14:58,166 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T19:14:58,167 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9baaea35-8dd3-4653-9b7c-03e5b792e644' 2024-12-04T19:14:58,167 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T19:14:58,167 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9baaea35-8dd3-4653-9b7c-03e5b792e644" 2024-12-04T19:14:58,167 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16e19285, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:14:58,167 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ba2734cd39f,46667,-1] 2024-12-04T19:14:58,167 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T19:14:58,168 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:14:58,169 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40490, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T19:14:58,169 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2746b2bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:14:58,170 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b10684e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:14:58,170 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ba2734cd39f,46667,-1 for getting cluster id 2024-12-04T19:14:58,170 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T19:14:58,171 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9baaea35-8dd3-4653-9b7c-03e5b792e644' 2024-12-04T19:14:58,172 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T19:14:58,172 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9baaea35-8dd3-4653-9b7c-03e5b792e644" 2024-12-04T19:14:58,172 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26b056db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:14:58,172 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ba2734cd39f,46667,-1] 2024-12-04T19:14:58,172 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T19:14:58,172 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:14:58,174 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40514, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T19:14:58,174 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ed27ba2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:14:58,175 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33f55aa1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:14:58,175 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ba2734cd39f,46667,-1 for getting cluster id 2024-12-04T19:14:58,175 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T19:14:58,176 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9baaea35-8dd3-4653-9b7c-03e5b792e644' 2024-12-04T19:14:58,176 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T19:14:58,177 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9baaea35-8dd3-4653-9b7c-03e5b792e644" 2024-12-04T19:14:58,177 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@706eb150, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:14:58,177 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ba2734cd39f,46667,-1] 2024-12-04T19:14:58,177 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T19:14:58,177 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:14:58,178 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40536, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T19:14:58,178 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ab9d92, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:14:58,179 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@558c177, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:14:58,179 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ba2734cd39f,46667,-1 for getting cluster id 2024-12-04T19:14:58,179 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T19:14:58,180 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9baaea35-8dd3-4653-9b7c-03e5b792e644' 2024-12-04T19:14:58,180 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T19:14:58,180 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9baaea35-8dd3-4653-9b7c-03e5b792e644" 2024-12-04T19:14:58,181 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d2196e1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:14:58,181 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ba2734cd39f,46667,-1] 2024-12-04T19:14:58,181 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T19:14:58,181 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:14:58,182 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40556, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T19:14:58,183 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d73212a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:14:58,183 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2df7df3b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:14:58,183 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ba2734cd39f,46667,-1 for getting cluster id 2024-12-04T19:14:58,183 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T19:14:58,184 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9baaea35-8dd3-4653-9b7c-03e5b792e644' 2024-12-04T19:14:58,184 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T19:14:58,184 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9baaea35-8dd3-4653-9b7c-03e5b792e644" 2024-12-04T19:14:58,184 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8bef5b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:14:58,184 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ba2734cd39f,46667,-1] 2024-12-04T19:14:58,185 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T19:14:58,185 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:14:58,185 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40580, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T19:14:58,186 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4628dcc0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:14:58,186 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10e1d32, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:14:58,186 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ba2734cd39f,46667,-1 for getting cluster id 2024-12-04T19:14:58,186 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T19:14:58,187 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9baaea35-8dd3-4653-9b7c-03e5b792e644' 2024-12-04T19:14:58,187 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T19:14:58,187 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9baaea35-8dd3-4653-9b7c-03e5b792e644" 2024-12-04T19:14:58,187 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5582711c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:14:58,187 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ba2734cd39f,46667,-1] 2024-12-04T19:14:58,187 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T19:14:58,187 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:14:58,188 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40600, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T19:14:58,188 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5afddb1d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:14:58,189 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@538e3e27, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:14:58,189 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ba2734cd39f,46667,-1 for getting cluster id 2024-12-04T19:14:58,189 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T19:14:58,189 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9baaea35-8dd3-4653-9b7c-03e5b792e644' 2024-12-04T19:14:58,190 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T19:14:58,190 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9baaea35-8dd3-4653-9b7c-03e5b792e644" 2024-12-04T19:14:58,190 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2311ecb4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:14:58,190 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ba2734cd39f,46667,-1] 2024-12-04T19:14:58,190 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T19:14:58,190 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:14:58,191 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40620, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T19:14:58,191 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c021c85, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:14:58,192 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67d38b7d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:14:58,192 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ba2734cd39f,46667,-1 for getting cluster id 2024-12-04T19:14:58,192 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T19:14:58,193 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9baaea35-8dd3-4653-9b7c-03e5b792e644' 2024-12-04T19:14:58,193 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T19:14:58,193 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9baaea35-8dd3-4653-9b7c-03e5b792e644" 2024-12-04T19:14:58,193 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ff54324, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:14:58,193 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ba2734cd39f,46667,-1] 2024-12-04T19:14:58,193 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T19:14:58,193 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:14:58,194 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40642, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T19:14:58,194 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63455081, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:14:58,195 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@314dc418, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:14:58,195 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ba2734cd39f,46667,-1 for getting cluster id 2024-12-04T19:14:58,195 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T19:14:58,196 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9baaea35-8dd3-4653-9b7c-03e5b792e644' 2024-12-04T19:14:58,196 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T19:14:58,196 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9baaea35-8dd3-4653-9b7c-03e5b792e644" 2024-12-04T19:14:58,196 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c009463, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:14:58,196 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ba2734cd39f,46667,-1] 2024-12-04T19:14:58,196 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T19:14:58,196 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:14:58,197 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40664, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T19:14:58,198 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4773470c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:14:58,198 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d27b054, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:14:58,198 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ba2734cd39f,46667,-1 for getting cluster id 2024-12-04T19:14:58,198 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T19:14:58,199 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9baaea35-8dd3-4653-9b7c-03e5b792e644' 2024-12-04T19:14:58,199 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T19:14:58,199 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9baaea35-8dd3-4653-9b7c-03e5b792e644" 2024-12-04T19:14:58,199 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75148191, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:14:58,199 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ba2734cd39f,46667,-1] 2024-12-04T19:14:58,199 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T19:14:58,199 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:14:58,200 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40678, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T19:14:58,201 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70d2c5d2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:14:58,201 DEBUG [Thread-1190 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T19:14:58,201 DEBUG [Thread-1192 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T19:14:58,201 DEBUG [Thread-1185 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T19:14:58,201 DEBUG [Thread-1198 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T19:14:58,201 DEBUG [Thread-1181 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T19:14:58,202 DEBUG [Thread-1196 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T19:14:58,202 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ba2734cd39f,33969,1733339629791, seqNum=-1] 2024-12-04T19:14:58,202 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ba2734cd39f,33969,1733339629791, seqNum=-1] 2024-12-04T19:14:58,202 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ba2734cd39f,33969,1733339629791, seqNum=-1] 2024-12-04T19:14:58,202 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ba2734cd39f,33969,1733339629791, seqNum=-1] 2024-12-04T19:14:58,202 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T19:14:58,202 DEBUG [Thread-1179 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T19:14:58,203 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T19:14:58,203 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T19:14:58,203 DEBUG [Thread-1183 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T19:14:58,203 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T19:14:58,203 DEBUG [Thread-1187 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T19:14:58,203 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ba2734cd39f,33969,1733339629791, seqNum=-1] 2024-12-04T19:14:58,204 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T19:14:58,204 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T19:14:58,204 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37176, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T19:14:58,204 DEBUG [Thread-1194 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T19:14:58,204 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ba2734cd39f,33969,1733339629791, seqNum=-1] 2024-12-04T19:14:58,204 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ba2734cd39f,33969,1733339629791, seqNum=-1] 2024-12-04T19:14:58,204 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T19:14:58,204 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T19:14:58,204 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37180, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T19:14:58,205 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37202, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T19:14:58,205 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37212, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T19:14:58,205 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37188, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T19:14:58,205 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ba2734cd39f,33969,1733339629791, seqNum=-1] 2024-12-04T19:14:58,205 DEBUG [Thread-1196 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ba2734cd39f,33969,1733339629791, seqNum=-1] 2024-12-04T19:14:58,205 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ba2734cd39f,33969,1733339629791, seqNum=-1] 2024-12-04T19:14:58,205 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37228, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T19:14:58,205 DEBUG [Thread-1196 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T19:14:58,205 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T19:14:58,205 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37220, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T19:14:58,205 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T19:14:58,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-12-04T19:14:58,207 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37238, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T19:14:58,207 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37246, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T19:14:58,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-04T19:14:58,207 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37250, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T19:14:58,207 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T19:14:58,207 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2] 2024-12-04T19:14:58,207 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2] 2024-12-04T19:14:58,208 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2] 2024-12-04T19:14:58,208 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2] 2024-12-04T19:14:58,209 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2] 2024-12-04T19:14:58,209 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T19:14:58,209 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T19:14:58,211 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2] 2024-12-04T19:14:58,211 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2] 2024-12-04T19:14:58,212 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2] 2024-12-04T19:14:58,212 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2] 2024-12-04T19:14:58,212 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2] 2024-12-04T19:14:58,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(8855): Flush requested on 6c5c1603a070441ac38abd9294c86a68 2024-12-04T19:14:58,221 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6c5c1603a070441ac38abd9294c86a68 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-04T19:14:58,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c5c1603a070441ac38abd9294c86a68, store=A 2024-12-04T19:14:58,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:14:58,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c5c1603a070441ac38abd9294c86a68, store=B 2024-12-04T19:14:58,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:14:58,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c5c1603a070441ac38abd9294c86a68, store=C 2024-12-04T19:14:58,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:14:58,249 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:58,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37176 deadline: 1733339758245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:58,250 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:58,250 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:58,250 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:58,252 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:58,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37212 deadline: 1733339758247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:58,253 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:58,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37250 deadline: 1733339758248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:58,253 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:58,253 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:58,253 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:58,253 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:58,253 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:58,253 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:58,254 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:58,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37220 deadline: 1733339758250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:58,254 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:58,254 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:58,254 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:58,254 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:58,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37228 deadline: 1733339758250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:58,254 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:58,254 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:58,254 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:58,255 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/A/a3d57a61ea6e43dab1cb710687593cd0 is 50, key is test_row_0/A:col10/1733339698218/Put/seqid=0 2024-12-04T19:14:58,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742080_1256 (size=14341) 2024-12-04T19:14:58,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-04T19:14:58,354 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:58,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37176 deadline: 1733339758351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:58,354 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:58,354 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:58,354 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:58,363 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-04T19:14:58,363 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:58,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. 2024-12-04T19:14:58,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37212 deadline: 1733339758362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:58,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. as already flushing 2024-12-04T19:14:58,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. 2024-12-04T19:14:58,363 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 6c5c1603a070441ac38abd9294c86a68, NAME => 'TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:58,363 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:58,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 6c5c1603a070441ac38abd9294c86a68, NAME => 'TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:58,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37250 deadline: 1733339758362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:58,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c5c1603a070441ac38abd9294c86a68, NAME => 'TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c5c1603a070441ac38abd9294c86a68, NAME => 'TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:58,364 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:58,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37220 deadline: 1733339758362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:58,365 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:58,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37228 deadline: 1733339758362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:58,365 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:58,365 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:58,365 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:58,365 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:58,365 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:58,365 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:58,365 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:58,365 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:58,365 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:58,365 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:58,365 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:58,365 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:58,517 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-04T19:14:58,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. 2024-12-04T19:14:58,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. as already flushing 2024-12-04T19:14:58,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. 2024-12-04T19:14:58,518 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 6c5c1603a070441ac38abd9294c86a68, NAME => 'TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:58,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 6c5c1603a070441ac38abd9294c86a68, NAME => 'TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:58,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c5c1603a070441ac38abd9294c86a68, NAME => 'TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c5c1603a070441ac38abd9294c86a68, NAME => 'TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:58,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-04T19:14:58,564 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:58,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37176 deadline: 1733339758562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:58,565 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:58,565 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:58,565 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:58,574 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:58,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37220 deadline: 1733339758572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:58,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:58,574 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:58,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:58,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:58,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37250 deadline: 1733339758572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:58,575 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:58,575 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:58,575 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:58,575 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:58,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37212 deadline: 1733339758572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:58,575 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:58,575 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:58,575 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:58,575 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:58,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37228 deadline: 1733339758572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:58,576 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:58,576 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:58,576 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:58,662 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/A/a3d57a61ea6e43dab1cb710687593cd0 2024-12-04T19:14:58,674 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-04T19:14:58,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. 2024-12-04T19:14:58,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. as already flushing 2024-12-04T19:14:58,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. 2024-12-04T19:14:58,674 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 6c5c1603a070441ac38abd9294c86a68, NAME => 'TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:58,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 6c5c1603a070441ac38abd9294c86a68, NAME => 'TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:58,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c5c1603a070441ac38abd9294c86a68, NAME => 'TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c5c1603a070441ac38abd9294c86a68, NAME => 'TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:58,687 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/B/6001cbf0269f439399b6b6c426147783 is 50, key is test_row_0/B:col10/1733339698218/Put/seqid=0 2024-12-04T19:14:58,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742081_1257 (size=12001) 2024-12-04T19:14:58,827 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-04T19:14:58,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. 2024-12-04T19:14:58,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. as already flushing 2024-12-04T19:14:58,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. 2024-12-04T19:14:58,828 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 6c5c1603a070441ac38abd9294c86a68, NAME => 'TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:58,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 6c5c1603a070441ac38abd9294c86a68, NAME => 'TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:58,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c5c1603a070441ac38abd9294c86a68, NAME => 'TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c5c1603a070441ac38abd9294c86a68, NAME => 'TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:58,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-04T19:14:58,873 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:58,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37176 deadline: 1733339758872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:58,874 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:58,874 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:58,874 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:58,883 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:58,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37220 deadline: 1733339758881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:58,883 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:58,883 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:58,883 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:58,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37212 deadline: 1733339758881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:58,883 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:58,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37250 deadline: 1733339758881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:58,883 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:58,884 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:58,883 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:58,884 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:58,884 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:58,884 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:58,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37228 deadline: 1733339758881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:58,884 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:58,884 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:58,884 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:58,884 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:58,884 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:58,980 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-04T19:14:58,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. 2024-12-04T19:14:58,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. as already flushing 2024-12-04T19:14:58,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. 2024-12-04T19:14:58,981 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 6c5c1603a070441ac38abd9294c86a68, NAME => 'TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:58,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 6c5c1603a070441ac38abd9294c86a68, NAME => 'TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:58,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c5c1603a070441ac38abd9294c86a68, NAME => 'TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c5c1603a070441ac38abd9294c86a68, NAME => 'TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:59,099 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/B/6001cbf0269f439399b6b6c426147783 2024-12-04T19:14:59,118 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/C/5d01b804b8504e3cbee56d8252ee14ee is 50, key is test_row_0/C:col10/1733339698218/Put/seqid=0 2024-12-04T19:14:59,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742082_1258 (size=12001) 2024-12-04T19:14:59,136 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-04T19:14:59,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. 2024-12-04T19:14:59,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. as already flushing 2024-12-04T19:14:59,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. 2024-12-04T19:14:59,136 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 6c5c1603a070441ac38abd9294c86a68, NAME => 'TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:59,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 6c5c1603a070441ac38abd9294c86a68, NAME => 'TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:59,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c5c1603a070441ac38abd9294c86a68, NAME => 'TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c5c1603a070441ac38abd9294c86a68, NAME => 'TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:59,289 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-04T19:14:59,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. 2024-12-04T19:14:59,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. as already flushing 2024-12-04T19:14:59,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. 2024-12-04T19:14:59,289 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 6c5c1603a070441ac38abd9294c86a68, NAME => 'TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:59,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 6c5c1603a070441ac38abd9294c86a68, NAME => 'TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:59,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c5c1603a070441ac38abd9294c86a68, NAME => 'TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c5c1603a070441ac38abd9294c86a68, NAME => 'TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:59,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-04T19:14:59,383 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:59,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37176 deadline: 1733339759381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:59,383 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:59,383 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:59,383 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:59,393 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:59,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37220 deadline: 1733339759391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:59,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:59,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:59,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:59,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:59,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37212 deadline: 1733339759391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:59,394 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:59,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37250 deadline: 1733339759391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:59,394 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:59,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:14:59,394 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:59,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37228 deadline: 1733339759392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:14:59,394 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:59,394 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:59,394 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:59,394 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:59,394 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:59,394 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:14:59,394 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:14:59,442 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-04T19:14:59,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. 2024-12-04T19:14:59,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. as already flushing 2024-12-04T19:14:59,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. 2024-12-04T19:14:59,442 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 6c5c1603a070441ac38abd9294c86a68, NAME => 'TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:59,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 6c5c1603a070441ac38abd9294c86a68, NAME => 'TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:59,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c5c1603a070441ac38abd9294c86a68, NAME => 'TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c5c1603a070441ac38abd9294c86a68, NAME => 'TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:14:59,523 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/C/5d01b804b8504e3cbee56d8252ee14ee 2024-12-04T19:14:59,527 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/A/a3d57a61ea6e43dab1cb710687593cd0 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/a3d57a61ea6e43dab1cb710687593cd0 2024-12-04T19:14:59,531 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/a3d57a61ea6e43dab1cb710687593cd0, entries=200, sequenceid=13, filesize=14.0 K 2024-12-04T19:14:59,532 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/B/6001cbf0269f439399b6b6c426147783 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/6001cbf0269f439399b6b6c426147783 2024-12-04T19:14:59,535 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/6001cbf0269f439399b6b6c426147783, entries=150, sequenceid=13, filesize=11.7 K 2024-12-04T19:14:59,536 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/C/5d01b804b8504e3cbee56d8252ee14ee as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/5d01b804b8504e3cbee56d8252ee14ee 2024-12-04T19:14:59,541 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/5d01b804b8504e3cbee56d8252ee14ee, entries=150, sequenceid=13, filesize=11.7 K 2024-12-04T19:14:59,542 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 6c5c1603a070441ac38abd9294c86a68 in 1321ms, sequenceid=13, compaction requested=false 2024-12-04T19:14:59,542 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6c5c1603a070441ac38abd9294c86a68: 2024-12-04T19:14:59,595 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-04T19:14:59,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. 2024-12-04T19:14:59,595 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing 6c5c1603a070441ac38abd9294c86a68 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-04T19:14:59,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c5c1603a070441ac38abd9294c86a68, store=A 2024-12-04T19:14:59,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:14:59,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c5c1603a070441ac38abd9294c86a68, store=B 2024-12-04T19:14:59,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:14:59,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c5c1603a070441ac38abd9294c86a68, store=C 2024-12-04T19:14:59,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:14:59,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/A/2465597156e04dd7a648a214d70358a9 is 50, key is test_row_0/A:col10/1733339698248/Put/seqid=0 2024-12-04T19:14:59,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742083_1259 (size=12001) 2024-12-04T19:14:59,843 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-04T19:15:00,006 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/A/2465597156e04dd7a648a214d70358a9 2024-12-04T19:15:00,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/B/31535e90f5cd40eb9507bad66eb08ce1 is 50, key is test_row_0/B:col10/1733339698248/Put/seqid=0 2024-12-04T19:15:00,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742084_1260 (size=12001) 2024-12-04T19:15:00,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-04T19:15:00,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(8855): Flush requested on 6c5c1603a070441ac38abd9294c86a68 2024-12-04T19:15:00,394 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. as already flushing 2024-12-04T19:15:00,407 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:00,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37220 deadline: 1733339760404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:00,408 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:00,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37212 deadline: 1733339760404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:00,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:00,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:00,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:00,408 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:00,408 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:00,408 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:00,408 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:00,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37250 deadline: 1733339760405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:00,408 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:00,408 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:00,408 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:00,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:00,409 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:00,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37228 deadline: 1733339760406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:00,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37176 deadline: 1733339760407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:00,410 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:00,410 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:00,410 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:00,410 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:00,410 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:00,410 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:00,417 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/B/31535e90f5cd40eb9507bad66eb08ce1 2024-12-04T19:15:00,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/C/6de385aac68a41ee84a839e1814a7292 is 50, key is test_row_0/C:col10/1733339698248/Put/seqid=0 2024-12-04T19:15:00,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742085_1261 (size=12001) 2024-12-04T19:15:00,513 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:00,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37220 deadline: 1733339760512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:00,514 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:00,513 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:00,514 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:00,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37212 deadline: 1733339760512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:00,514 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:00,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:00,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:00,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:00,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:00,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37250 deadline: 1733339760512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:00,514 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:00,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:00,514 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:00,514 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:00,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37228 deadline: 1733339760512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:00,514 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:00,514 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:00,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:00,514 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:00,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37176 deadline: 1733339760513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:00,515 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:00,515 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:00,515 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:00,724 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:00,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37220 deadline: 1733339760722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:00,724 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:00,724 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:00,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37212 deadline: 1733339760722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:00,724 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:00,724 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:00,725 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:00,725 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:00,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37250 deadline: 1733339760722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:00,725 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:00,725 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:00,725 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:00,725 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:00,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37228 deadline: 1733339760722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:00,725 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:00,725 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:00,725 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:00,725 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:00,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37176 deadline: 1733339760723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:00,725 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:00,725 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:00,725 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:00,725 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:00,725 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:00,830 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/C/6de385aac68a41ee84a839e1814a7292 2024-12-04T19:15:00,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/A/2465597156e04dd7a648a214d70358a9 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/2465597156e04dd7a648a214d70358a9 2024-12-04T19:15:00,839 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/2465597156e04dd7a648a214d70358a9, entries=150, sequenceid=33, filesize=11.7 K 2024-12-04T19:15:00,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/B/31535e90f5cd40eb9507bad66eb08ce1 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/31535e90f5cd40eb9507bad66eb08ce1 2024-12-04T19:15:00,843 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/31535e90f5cd40eb9507bad66eb08ce1, entries=150, sequenceid=33, filesize=11.7 K 2024-12-04T19:15:00,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/C/6de385aac68a41ee84a839e1814a7292 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/6de385aac68a41ee84a839e1814a7292 2024-12-04T19:15:00,849 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/6de385aac68a41ee84a839e1814a7292, entries=150, sequenceid=33, filesize=11.7 K 2024-12-04T19:15:00,850 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 6c5c1603a070441ac38abd9294c86a68 in 1254ms, sequenceid=33, compaction requested=false 2024-12-04T19:15:00,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for 6c5c1603a070441ac38abd9294c86a68: 2024-12-04T19:15:00,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. 2024-12-04T19:15:00,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-12-04T19:15:00,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-12-04T19:15:00,853 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-12-04T19:15:00,853 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6420 sec 2024-12-04T19:15:00,856 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 2.6520 sec 2024-12-04T19:15:01,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(8855): Flush requested on 6c5c1603a070441ac38abd9294c86a68 2024-12-04T19:15:01,034 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6c5c1603a070441ac38abd9294c86a68 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-04T19:15:01,035 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c5c1603a070441ac38abd9294c86a68, store=A 2024-12-04T19:15:01,035 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:15:01,035 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c5c1603a070441ac38abd9294c86a68, store=B 2024-12-04T19:15:01,035 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:15:01,035 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c5c1603a070441ac38abd9294c86a68, store=C 2024-12-04T19:15:01,035 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:15:01,040 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/A/41c356f67a9543d5ae9128d5d78580dd is 50, key is test_row_0/A:col10/1733339701034/Put/seqid=0 2024-12-04T19:15:01,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742086_1262 (size=12001) 2024-12-04T19:15:01,073 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:01,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37228 deadline: 1733339761068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:01,074 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:01,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37212 deadline: 1733339761069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:01,074 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:01,074 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:01,074 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:01,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37176 deadline: 1733339761069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:01,074 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:01,074 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:01,074 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:01,074 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:01,074 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:01,074 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:01,074 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:01,077 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:01,077 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:01,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37220 deadline: 1733339761073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:01,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37250 deadline: 1733339761074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:01,077 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:01,077 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:01,077 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:01,077 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:01,077 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:01,077 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:01,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:01,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37228 deadline: 1733339761181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:01,184 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:01,184 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:01,184 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:01,184 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:01,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37212 deadline: 1733339761181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:01,184 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:01,184 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:01,184 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:01,184 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:01,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37176 deadline: 1733339761182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:01,184 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:01,184 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:01,184 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:01,184 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:01,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37250 deadline: 1733339761182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:01,185 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:01,185 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:01,185 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:01,185 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:01,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37220 deadline: 1733339761182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:01,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:01,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:01,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:01,393 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:01,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37228 deadline: 1733339761391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:01,393 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:01,393 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:01,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37212 deadline: 1733339761391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:01,393 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:01,393 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:01,393 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:01,393 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:01,394 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:01,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37176 deadline: 1733339761392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:01,394 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:01,394 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:01,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:01,394 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:01,394 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:01,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37250 deadline: 1733339761392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:01,394 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:01,394 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:01,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:01,394 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:01,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37220 deadline: 1733339761392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:01,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:01,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:01,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:01,444 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/A/41c356f67a9543d5ae9128d5d78580dd 2024-12-04T19:15:01,450 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/B/8ad544a1e66a44bd88fcc357cdff4f66 is 50, key is test_row_0/B:col10/1733339701034/Put/seqid=0 2024-12-04T19:15:01,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742087_1263 (size=12001) 2024-12-04T19:15:01,704 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:01,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37228 deadline: 1733339761702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:01,704 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:01,704 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:01,704 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:01,704 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:01,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37212 deadline: 1733339761702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:01,704 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:01,704 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:01,705 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:01,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37176 deadline: 1733339761702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:01,705 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:01,705 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:01,705 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:01,705 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:01,705 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:01,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37250 deadline: 1733339761702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:01,705 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:01,705 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:01,705 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:01,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37220 deadline: 1733339761702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:01,705 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:01,705 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:01,705 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:01,705 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:01,854 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/B/8ad544a1e66a44bd88fcc357cdff4f66 2024-12-04T19:15:01,870 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/C/9aaa03f052ef477c9ca586e06c9fb4ff is 50, key is test_row_0/C:col10/1733339701034/Put/seqid=0 2024-12-04T19:15:01,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742088_1264 (size=12001) 2024-12-04T19:15:02,214 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:02,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37228 deadline: 1733339762211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:02,215 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:02,215 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:02,215 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:02,215 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:02,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37212 deadline: 1733339762211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:02,215 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:02,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37176 deadline: 1733339762211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:02,215 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:02,215 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:02,215 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:02,215 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:02,215 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:02,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37250 deadline: 1733339762212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:02,215 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:02,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37220 deadline: 1733339762212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:02,216 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:02,216 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:02,216 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:02,216 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:02,216 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:02,216 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:02,216 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:02,216 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:02,295 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/C/9aaa03f052ef477c9ca586e06c9fb4ff 2024-12-04T19:15:02,306 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/A/41c356f67a9543d5ae9128d5d78580dd as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/41c356f67a9543d5ae9128d5d78580dd 2024-12-04T19:15:02,309 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/41c356f67a9543d5ae9128d5d78580dd, entries=150, sequenceid=48, filesize=11.7 K 2024-12-04T19:15:02,310 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/B/8ad544a1e66a44bd88fcc357cdff4f66 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/8ad544a1e66a44bd88fcc357cdff4f66 2024-12-04T19:15:02,313 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/8ad544a1e66a44bd88fcc357cdff4f66, entries=150, sequenceid=48, filesize=11.7 K 2024-12-04T19:15:02,314 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/C/9aaa03f052ef477c9ca586e06c9fb4ff as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/9aaa03f052ef477c9ca586e06c9fb4ff 2024-12-04T19:15:02,318 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/9aaa03f052ef477c9ca586e06c9fb4ff, entries=150, sequenceid=48, filesize=11.7 K 2024-12-04T19:15:02,318 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 6c5c1603a070441ac38abd9294c86a68 in 1284ms, sequenceid=48, compaction requested=true 2024-12-04T19:15:02,319 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6c5c1603a070441ac38abd9294c86a68: 2024-12-04T19:15:02,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6c5c1603a070441ac38abd9294c86a68:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T19:15:02,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:15:02,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6c5c1603a070441ac38abd9294c86a68:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T19:15:02,319 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T19:15:02,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:15:02,319 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T19:15:02,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6c5c1603a070441ac38abd9294c86a68:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T19:15:02,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T19:15:02,320 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T19:15:02,320 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T19:15:02,320 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1541): 6c5c1603a070441ac38abd9294c86a68/B is initiating minor compaction (all files) 2024-12-04T19:15:02,320 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1541): 6c5c1603a070441ac38abd9294c86a68/A is initiating minor compaction (all files) 2024-12-04T19:15:02,320 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6c5c1603a070441ac38abd9294c86a68/A in TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. 2024-12-04T19:15:02,320 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6c5c1603a070441ac38abd9294c86a68/B in TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. 2024-12-04T19:15:02,320 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/6001cbf0269f439399b6b6c426147783, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/31535e90f5cd40eb9507bad66eb08ce1, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/8ad544a1e66a44bd88fcc357cdff4f66] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp, totalSize=35.2 K 2024-12-04T19:15:02,320 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/a3d57a61ea6e43dab1cb710687593cd0, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/2465597156e04dd7a648a214d70358a9, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/41c356f67a9543d5ae9128d5d78580dd] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp, totalSize=37.4 K 2024-12-04T19:15:02,320 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting a3d57a61ea6e43dab1cb710687593cd0, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733339698213 2024-12-04T19:15:02,320 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 6001cbf0269f439399b6b6c426147783, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733339698213 2024-12-04T19:15:02,321 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2465597156e04dd7a648a214d70358a9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1733339698245 2024-12-04T19:15:02,321 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 31535e90f5cd40eb9507bad66eb08ce1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1733339698245 2024-12-04T19:15:02,321 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 41c356f67a9543d5ae9128d5d78580dd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1733339701033 2024-12-04T19:15:02,322 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ad544a1e66a44bd88fcc357cdff4f66, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1733339701033 2024-12-04T19:15:02,333 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6c5c1603a070441ac38abd9294c86a68#A#compaction#219 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:15:02,334 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/A/5ccd8ab462434030b34f7d2b2d48df18 is 50, key is test_row_0/A:col10/1733339701034/Put/seqid=0 2024-12-04T19:15:02,336 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6c5c1603a070441ac38abd9294c86a68#B#compaction#220 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:15:02,337 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/B/a0a65f53492e415281cd487ae9a8df53 is 50, key is test_row_0/B:col10/1733339701034/Put/seqid=0 2024-12-04T19:15:02,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742090_1266 (size=12104) 2024-12-04T19:15:02,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742089_1265 (size=12104) 2024-12-04T19:15:02,361 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/A/5ccd8ab462434030b34f7d2b2d48df18 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/5ccd8ab462434030b34f7d2b2d48df18 2024-12-04T19:15:02,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-04T19:15:02,373 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T19:15:02,374 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T19:15:02,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-12-04T19:15:02,376 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6c5c1603a070441ac38abd9294c86a68/A of 6c5c1603a070441ac38abd9294c86a68 into 5ccd8ab462434030b34f7d2b2d48df18(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:15:02,376 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6c5c1603a070441ac38abd9294c86a68: 2024-12-04T19:15:02,376 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., storeName=6c5c1603a070441ac38abd9294c86a68/A, priority=13, startTime=1733339702319; duration=0sec 2024-12-04T19:15:02,376 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T19:15:02,376 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T19:15:02,376 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6c5c1603a070441ac38abd9294c86a68:A 2024-12-04T19:15:02,376 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T19:15:02,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-04T19:15:02,378 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T19:15:02,378 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T19:15:02,378 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T19:15:02,379 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1541): 6c5c1603a070441ac38abd9294c86a68/C is initiating minor compaction (all files) 2024-12-04T19:15:02,379 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6c5c1603a070441ac38abd9294c86a68/C in TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. 2024-12-04T19:15:02,379 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/5d01b804b8504e3cbee56d8252ee14ee, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/6de385aac68a41ee84a839e1814a7292, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/9aaa03f052ef477c9ca586e06c9fb4ff] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp, totalSize=35.2 K 2024-12-04T19:15:02,380 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5d01b804b8504e3cbee56d8252ee14ee, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733339698213 2024-12-04T19:15:02,382 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6de385aac68a41ee84a839e1814a7292, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1733339698245 2024-12-04T19:15:02,385 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9aaa03f052ef477c9ca586e06c9fb4ff, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1733339701033 2024-12-04T19:15:02,410 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6c5c1603a070441ac38abd9294c86a68#C#compaction#221 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:15:02,411 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/C/afe53e5d74ec41d182bcd3d1ef0c60a8 is 50, key is test_row_0/C:col10/1733339701034/Put/seqid=0 2024-12-04T19:15:02,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742091_1267 (size=12104) 2024-12-04T19:15:02,432 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/C/afe53e5d74ec41d182bcd3d1ef0c60a8 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/afe53e5d74ec41d182bcd3d1ef0c60a8 2024-12-04T19:15:02,438 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6c5c1603a070441ac38abd9294c86a68/C of 6c5c1603a070441ac38abd9294c86a68 into afe53e5d74ec41d182bcd3d1ef0c60a8(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:15:02,438 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6c5c1603a070441ac38abd9294c86a68: 2024-12-04T19:15:02,438 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., storeName=6c5c1603a070441ac38abd9294c86a68/C, priority=13, startTime=1733339702319; duration=0sec 2024-12-04T19:15:02,439 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:15:02,439 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6c5c1603a070441ac38abd9294c86a68:C 2024-12-04T19:15:02,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-04T19:15:02,531 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-04T19:15:02,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. 2024-12-04T19:15:02,531 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing 6c5c1603a070441ac38abd9294c86a68 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-04T19:15:02,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c5c1603a070441ac38abd9294c86a68, store=A 2024-12-04T19:15:02,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:15:02,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c5c1603a070441ac38abd9294c86a68, store=B 2024-12-04T19:15:02,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:15:02,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c5c1603a070441ac38abd9294c86a68, store=C 2024-12-04T19:15:02,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:15:02,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/A/d45b55e78f644e5dabaf814d41f4e035 is 50, key is test_row_0/A:col10/1733339701072/Put/seqid=0 2024-12-04T19:15:02,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742092_1268 (size=12001) 2024-12-04T19:15:02,544 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=67 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/A/d45b55e78f644e5dabaf814d41f4e035 2024-12-04T19:15:02,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/B/d5929233b4b34345a5ba446e1a55d13e is 50, key is test_row_0/B:col10/1733339701072/Put/seqid=0 2024-12-04T19:15:02,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742093_1269 (size=12001) 2024-12-04T19:15:02,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-04T19:15:02,750 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/B/a0a65f53492e415281cd487ae9a8df53 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/a0a65f53492e415281cd487ae9a8df53 2024-12-04T19:15:02,754 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6c5c1603a070441ac38abd9294c86a68/B of 6c5c1603a070441ac38abd9294c86a68 into a0a65f53492e415281cd487ae9a8df53(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:15:02,754 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6c5c1603a070441ac38abd9294c86a68: 2024-12-04T19:15:02,754 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., storeName=6c5c1603a070441ac38abd9294c86a68/B, priority=13, startTime=1733339702319; duration=0sec 2024-12-04T19:15:02,755 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:15:02,755 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6c5c1603a070441ac38abd9294c86a68:B 2024-12-04T19:15:02,986 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=67 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/B/d5929233b4b34345a5ba446e1a55d13e 2024-12-04T19:15:03,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-04T19:15:03,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/C/0beffc7e4b05492398e612bc8763d934 is 50, key is test_row_0/C:col10/1733339701072/Put/seqid=0 2024-12-04T19:15:03,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742094_1270 (size=12001) 2024-12-04T19:15:03,031 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=67 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/C/0beffc7e4b05492398e612bc8763d934 2024-12-04T19:15:03,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/A/d45b55e78f644e5dabaf814d41f4e035 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/d45b55e78f644e5dabaf814d41f4e035 2024-12-04T19:15:03,053 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/d45b55e78f644e5dabaf814d41f4e035, entries=150, sequenceid=67, filesize=11.7 K 2024-12-04T19:15:03,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/B/d5929233b4b34345a5ba446e1a55d13e as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/d5929233b4b34345a5ba446e1a55d13e 2024-12-04T19:15:03,059 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/d5929233b4b34345a5ba446e1a55d13e, entries=150, sequenceid=67, filesize=11.7 K 2024-12-04T19:15:03,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/C/0beffc7e4b05492398e612bc8763d934 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/0beffc7e4b05492398e612bc8763d934 2024-12-04T19:15:03,068 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/0beffc7e4b05492398e612bc8763d934, entries=150, sequenceid=67, filesize=11.7 K 2024-12-04T19:15:03,070 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=0 B/0 for 6c5c1603a070441ac38abd9294c86a68 in 538ms, sequenceid=67, compaction requested=false 2024-12-04T19:15:03,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for 6c5c1603a070441ac38abd9294c86a68: 2024-12-04T19:15:03,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. 2024-12-04T19:15:03,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-12-04T19:15:03,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-12-04T19:15:03,074 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-12-04T19:15:03,074 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 694 msec 2024-12-04T19:15:03,077 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 702 msec 2024-12-04T19:15:03,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(8855): Flush requested on 6c5c1603a070441ac38abd9294c86a68 2024-12-04T19:15:03,237 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6c5c1603a070441ac38abd9294c86a68 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-04T19:15:03,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c5c1603a070441ac38abd9294c86a68, store=A 2024-12-04T19:15:03,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:15:03,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c5c1603a070441ac38abd9294c86a68, store=B 2024-12-04T19:15:03,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:15:03,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c5c1603a070441ac38abd9294c86a68, store=C 2024-12-04T19:15:03,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:15:03,248 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/A/4ecc07e726d045dd8e95c474560b5d54 is 50, key is test_row_0/A:col10/1733339703229/Put/seqid=0 2024-12-04T19:15:03,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742095_1271 (size=12001) 2024-12-04T19:15:03,274 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/A/4ecc07e726d045dd8e95c474560b5d54 2024-12-04T19:15:03,283 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:03,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37228 deadline: 1733339763275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:03,284 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:03,284 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:03,284 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:03,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37220 deadline: 1733339763278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:03,284 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:03,284 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:03,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37176 deadline: 1733339763278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:03,285 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:03,285 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:03,285 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:03,285 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:03,285 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:03,285 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:03,288 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:03,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37212 deadline: 1733339763280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:03,289 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:03,289 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:03,289 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:03,289 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:03,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37250 deadline: 1733339763283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:03,289 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:03,289 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:03,289 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:03,298 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/B/252d2957b54c467fa12eff14525362ac is 50, key is test_row_0/B:col10/1733339703229/Put/seqid=0 2024-12-04T19:15:03,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742096_1272 (size=12001) 2024-12-04T19:15:03,358 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/B/252d2957b54c467fa12eff14525362ac 2024-12-04T19:15:03,393 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:03,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37220 deadline: 1733339763392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:03,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:03,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:03,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:03,395 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:03,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37228 deadline: 1733339763392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:03,395 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:03,395 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:03,396 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:03,395 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:03,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37212 deadline: 1733339763392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:03,396 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:03,396 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:03,396 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:03,396 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:03,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37176 deadline: 1733339763392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:03,396 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:03,396 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:03,396 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:03,396 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:03,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37250 deadline: 1733339763392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:03,397 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:03,397 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:03,397 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:03,438 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/C/e29f983fbbd24a19b76696ecca48b866 is 50, key is test_row_0/C:col10/1733339703229/Put/seqid=0 2024-12-04T19:15:03,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742097_1273 (size=12001) 2024-12-04T19:15:03,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-04T19:15:03,512 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T19:15:03,512 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T19:15:03,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-12-04T19:15:03,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-04T19:15:03,514 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T19:15:03,515 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T19:15:03,515 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T19:15:03,604 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:03,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37220 deadline: 1733339763602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:03,604 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:03,604 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:03,604 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:03,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37212 deadline: 1733339763602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:03,604 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:03,604 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:03,604 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:03,605 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:03,605 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:03,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37228 deadline: 1733339763602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:03,605 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:03,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:03,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:03,605 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:03,605 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:03,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37176 deadline: 1733339763602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:03,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37250 deadline: 1733339763602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:03,605 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:03,605 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:03,605 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:03,605 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:03,605 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:03,605 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:03,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-04T19:15:03,680 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-04T19:15:03,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. 2024-12-04T19:15:03,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. as already flushing 2024-12-04T19:15:03,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. 2024-12-04T19:15:03,681 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 6c5c1603a070441ac38abd9294c86a68, NAME => 'TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:03,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 6c5c1603a070441ac38abd9294c86a68, NAME => 'TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:03,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c5c1603a070441ac38abd9294c86a68, NAME => 'TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c5c1603a070441ac38abd9294c86a68, NAME => 'TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:03,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-04T19:15:03,834 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-04T19:15:03,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. 2024-12-04T19:15:03,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. as already flushing 2024-12-04T19:15:03,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. 2024-12-04T19:15:03,834 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 6c5c1603a070441ac38abd9294c86a68, NAME => 'TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:03,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 6c5c1603a070441ac38abd9294c86a68, NAME => 'TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:03,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c5c1603a070441ac38abd9294c86a68, NAME => 'TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c5c1603a070441ac38abd9294c86a68, NAME => 'TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:03,871 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/C/e29f983fbbd24a19b76696ecca48b866 2024-12-04T19:15:03,875 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/A/4ecc07e726d045dd8e95c474560b5d54 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/4ecc07e726d045dd8e95c474560b5d54 2024-12-04T19:15:03,879 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/4ecc07e726d045dd8e95c474560b5d54, entries=150, sequenceid=80, filesize=11.7 K 2024-12-04T19:15:03,880 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/B/252d2957b54c467fa12eff14525362ac as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/252d2957b54c467fa12eff14525362ac 2024-12-04T19:15:03,883 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/252d2957b54c467fa12eff14525362ac, entries=150, sequenceid=80, filesize=11.7 K 2024-12-04T19:15:03,884 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/C/e29f983fbbd24a19b76696ecca48b866 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/e29f983fbbd24a19b76696ecca48b866 2024-12-04T19:15:03,888 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/e29f983fbbd24a19b76696ecca48b866, entries=150, sequenceid=80, filesize=11.7 K 2024-12-04T19:15:03,889 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 6c5c1603a070441ac38abd9294c86a68 in 653ms, sequenceid=80, compaction requested=true 2024-12-04T19:15:03,889 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6c5c1603a070441ac38abd9294c86a68: 2024-12-04T19:15:03,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6c5c1603a070441ac38abd9294c86a68:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T19:15:03,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:15:03,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6c5c1603a070441ac38abd9294c86a68:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T19:15:03,889 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T19:15:03,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:15:03,889 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T19:15:03,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6c5c1603a070441ac38abd9294c86a68:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T19:15:03,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T19:15:03,890 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T19:15:03,890 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T19:15:03,890 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1541): 6c5c1603a070441ac38abd9294c86a68/B is initiating minor compaction (all files) 2024-12-04T19:15:03,890 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1541): 6c5c1603a070441ac38abd9294c86a68/A is initiating minor compaction (all files) 2024-12-04T19:15:03,890 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6c5c1603a070441ac38abd9294c86a68/B in TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. 2024-12-04T19:15:03,890 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6c5c1603a070441ac38abd9294c86a68/A in TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. 2024-12-04T19:15:03,890 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/a0a65f53492e415281cd487ae9a8df53, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/d5929233b4b34345a5ba446e1a55d13e, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/252d2957b54c467fa12eff14525362ac] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp, totalSize=35.3 K 2024-12-04T19:15:03,890 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/5ccd8ab462434030b34f7d2b2d48df18, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/d45b55e78f644e5dabaf814d41f4e035, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/4ecc07e726d045dd8e95c474560b5d54] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp, totalSize=35.3 K 2024-12-04T19:15:03,891 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ccd8ab462434030b34f7d2b2d48df18, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1733339701033 2024-12-04T19:15:03,891 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting a0a65f53492e415281cd487ae9a8df53, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1733339701033 2024-12-04T19:15:03,891 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting d5929233b4b34345a5ba446e1a55d13e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=67, earliestPutTs=1733339701068 2024-12-04T19:15:03,891 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting d45b55e78f644e5dabaf814d41f4e035, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=67, earliestPutTs=1733339701068 2024-12-04T19:15:03,891 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 252d2957b54c467fa12eff14525362ac, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1733339703224 2024-12-04T19:15:03,891 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ecc07e726d045dd8e95c474560b5d54, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1733339703224 2024-12-04T19:15:03,899 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6c5c1603a070441ac38abd9294c86a68#A#compaction#228 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:15:03,900 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/A/27d763afc46340a5a5d4f541d357e7a0 is 50, key is test_row_0/A:col10/1733339703229/Put/seqid=0 2024-12-04T19:15:03,910 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6c5c1603a070441ac38abd9294c86a68#B#compaction#229 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:15:03,910 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/B/eacc9abf7b5d407988a54784987a450d is 50, key is test_row_0/B:col10/1733339703229/Put/seqid=0 2024-12-04T19:15:03,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742098_1274 (size=12207) 2024-12-04T19:15:03,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742099_1275 (size=12207) 2024-12-04T19:15:03,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(8855): Flush requested on 6c5c1603a070441ac38abd9294c86a68 2024-12-04T19:15:03,915 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6c5c1603a070441ac38abd9294c86a68 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-04T19:15:03,916 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733339703264, max timestamp=1733339703914Num uniques 859; 2024-12-04T19:15:03,916 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T19:15:03,916 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733339703264, max timestamp=1733339703914Num uniques 859; 2024-12-04T19:15:03,916 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T19:15:03,917 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733339703264, max timestamp=1733339703914Num uniques 859; 2024-12-04T19:15:03,917 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T19:15:03,918 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c5c1603a070441ac38abd9294c86a68, store=A 2024-12-04T19:15:03,918 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T19:15:03,918 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c5c1603a070441ac38abd9294c86a68, store=B 2024-12-04T19:15:03,918 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T19:15:03,918 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c5c1603a070441ac38abd9294c86a68, store=C 2024-12-04T19:15:03,918 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T19:15:03,922 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/A/21ea5931056f45d08b8348e5c21f67dd is 50, key is test_row_0/A:col10/1733339703915/Put/seqid=0 2024-12-04T19:15:03,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742100_1276 (size=12001) 2024-12-04T19:15:03,941 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1733339703930, max timestamp=1733339703939Num uniques 859; 2024-12-04T19:15:03,941 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-12-04T19:15:03,941 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1733339703930, max timestamp=1733339703939Num uniques 859; 2024-12-04T19:15:03,941 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1733339703930, max timestamp=1733339703939Num uniques 859; 2024-12-04T19:15:03,941 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-12-04T19:15:03,941 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-12-04T19:15:03,977 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:03,977 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:03,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37212 deadline: 1733339763954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:03,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37220 deadline: 1733339763953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:03,977 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:03,977 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:03,977 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:03,977 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:03,977 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:03,977 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:03,983 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:03,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37176 deadline: 1733339763977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:03,984 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:03,984 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:03,984 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:03,984 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:03,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37228 deadline: 1733339763977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:03,984 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:03,984 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:03,984 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:03,984 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:03,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37250 deadline: 1733339763977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:03,984 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:03,985 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:03,985 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:03,987 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-04T19:15:03,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. 2024-12-04T19:15:03,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. as already flushing 2024-12-04T19:15:03,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. 2024-12-04T19:15:03,987 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 6c5c1603a070441ac38abd9294c86a68, NAME => 'TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:03,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 6c5c1603a070441ac38abd9294c86a68, NAME => 'TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:03,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c5c1603a070441ac38abd9294c86a68, NAME => 'TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c5c1603a070441ac38abd9294c86a68, NAME => 'TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:04,084 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:04,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37220 deadline: 1733339764081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:04,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:04,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:04,085 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:04,084 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:04,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37212 deadline: 1733339764081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:04,085 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:04,085 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:04,085 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:04,093 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:04,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37176 deadline: 1733339764092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:04,094 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:04,094 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:04,094 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:04,094 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:04,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37228 deadline: 1733339764092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:04,094 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:04,094 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:04,094 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:04,095 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:04,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37250 deadline: 1733339764092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:04,095 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:04,095 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:04,095 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:04,139 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-04T19:15:04,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. 2024-12-04T19:15:04,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. as already flushing 2024-12-04T19:15:04,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. 2024-12-04T19:15:04,140 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 6c5c1603a070441ac38abd9294c86a68, NAME => 'TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:04,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 6c5c1603a070441ac38abd9294c86a68, NAME => 'TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:04,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c5c1603a070441ac38abd9294c86a68, NAME => 'TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c5c1603a070441ac38abd9294c86a68, NAME => 'TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:04,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-04T19:15:04,292 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-04T19:15:04,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. 2024-12-04T19:15:04,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. as already flushing 2024-12-04T19:15:04,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. 2024-12-04T19:15:04,293 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 6c5c1603a070441ac38abd9294c86a68, NAME => 'TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:04,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 6c5c1603a070441ac38abd9294c86a68, NAME => 'TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:04,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c5c1603a070441ac38abd9294c86a68, NAME => 'TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c5c1603a070441ac38abd9294c86a68, NAME => 'TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:04,295 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:04,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37212 deadline: 1733339764292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:04,295 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:04,295 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:04,295 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:04,295 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:04,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37220 deadline: 1733339764292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:04,295 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:04,296 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:04,296 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:04,306 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:04,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37176 deadline: 1733339764302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:04,306 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:04,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37228 deadline: 1733339764302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:04,306 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:04,306 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:04,306 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:04,306 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:04,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37250 deadline: 1733339764302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:04,307 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:04,307 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:04,307 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:04,307 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:04,307 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:04,307 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:04,316 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/A/27d763afc46340a5a5d4f541d357e7a0 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/27d763afc46340a5a5d4f541d357e7a0 2024-12-04T19:15:04,320 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6c5c1603a070441ac38abd9294c86a68/A of 6c5c1603a070441ac38abd9294c86a68 into 27d763afc46340a5a5d4f541d357e7a0(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:15:04,320 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6c5c1603a070441ac38abd9294c86a68: 2024-12-04T19:15:04,320 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., storeName=6c5c1603a070441ac38abd9294c86a68/A, priority=13, startTime=1733339703889; duration=0sec 2024-12-04T19:15:04,320 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T19:15:04,320 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6c5c1603a070441ac38abd9294c86a68:A 2024-12-04T19:15:04,321 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T19:15:04,321 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/B/eacc9abf7b5d407988a54784987a450d as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/eacc9abf7b5d407988a54784987a450d 2024-12-04T19:15:04,322 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T19:15:04,322 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1541): 6c5c1603a070441ac38abd9294c86a68/C is initiating minor compaction (all files) 2024-12-04T19:15:04,322 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6c5c1603a070441ac38abd9294c86a68/C in TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. 2024-12-04T19:15:04,322 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/afe53e5d74ec41d182bcd3d1ef0c60a8, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/0beffc7e4b05492398e612bc8763d934, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/e29f983fbbd24a19b76696ecca48b866] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp, totalSize=35.3 K 2024-12-04T19:15:04,322 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting afe53e5d74ec41d182bcd3d1ef0c60a8, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1733339701033 2024-12-04T19:15:04,323 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0beffc7e4b05492398e612bc8763d934, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=67, earliestPutTs=1733339701068 2024-12-04T19:15:04,323 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting e29f983fbbd24a19b76696ecca48b866, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1733339703224 2024-12-04T19:15:04,326 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6c5c1603a070441ac38abd9294c86a68/B of 6c5c1603a070441ac38abd9294c86a68 into eacc9abf7b5d407988a54784987a450d(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:15:04,326 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6c5c1603a070441ac38abd9294c86a68: 2024-12-04T19:15:04,326 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., storeName=6c5c1603a070441ac38abd9294c86a68/B, priority=13, startTime=1733339703889; duration=0sec 2024-12-04T19:15:04,326 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:15:04,326 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6c5c1603a070441ac38abd9294c86a68:B 2024-12-04T19:15:04,326 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/A/21ea5931056f45d08b8348e5c21f67dd 2024-12-04T19:15:04,329 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6c5c1603a070441ac38abd9294c86a68#C#compaction#231 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:15:04,330 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/C/f126694142444ca3a70c7dc9e9890a63 is 50, key is test_row_0/C:col10/1733339703229/Put/seqid=0 2024-12-04T19:15:04,335 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/B/e62474a74061431295169a9ec26831c5 is 50, key is test_row_0/B:col10/1733339703915/Put/seqid=0 2024-12-04T19:15:04,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742101_1277 (size=12207) 2024-12-04T19:15:04,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742102_1278 (size=12001) 2024-12-04T19:15:04,342 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/B/e62474a74061431295169a9ec26831c5 2024-12-04T19:15:04,349 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/C/53e3c78d791a4358951dd2e96f5133a3 is 50, key is test_row_0/C:col10/1733339703915/Put/seqid=0 2024-12-04T19:15:04,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742103_1279 (size=12001) 2024-12-04T19:15:04,445 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-04T19:15:04,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. 2024-12-04T19:15:04,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. as already flushing 2024-12-04T19:15:04,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. 2024-12-04T19:15:04,445 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 6c5c1603a070441ac38abd9294c86a68, NAME => 'TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:04,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 6c5c1603a070441ac38abd9294c86a68, NAME => 'TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:04,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c5c1603a070441ac38abd9294c86a68, NAME => 'TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c5c1603a070441ac38abd9294c86a68, NAME => 'TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:04,598 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-04T19:15:04,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. 2024-12-04T19:15:04,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. as already flushing 2024-12-04T19:15:04,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. 2024-12-04T19:15:04,598 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 6c5c1603a070441ac38abd9294c86a68, NAME => 'TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:04,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 6c5c1603a070441ac38abd9294c86a68, NAME => 'TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:04,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c5c1603a070441ac38abd9294c86a68, NAME => 'TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c5c1603a070441ac38abd9294c86a68, NAME => 'TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:04,604 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:04,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37212 deadline: 1733339764601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:04,604 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:04,604 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:04,604 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:04,604 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:04,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37220 deadline: 1733339764601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:04,605 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:04,605 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:04,605 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:04,615 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:04,615 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:04,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37176 deadline: 1733339764612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:04,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37228 deadline: 1733339764612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:04,616 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:04,616 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:04,616 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:04,616 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:04,616 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:04,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37250 deadline: 1733339764612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:04,616 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:04,616 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:04,616 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:04,616 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:04,616 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:04,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-04T19:15:04,741 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/C/f126694142444ca3a70c7dc9e9890a63 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/f126694142444ca3a70c7dc9e9890a63 2024-12-04T19:15:04,744 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6c5c1603a070441ac38abd9294c86a68/C of 6c5c1603a070441ac38abd9294c86a68 into f126694142444ca3a70c7dc9e9890a63(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:15:04,744 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6c5c1603a070441ac38abd9294c86a68: 2024-12-04T19:15:04,744 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., storeName=6c5c1603a070441ac38abd9294c86a68/C, priority=13, startTime=1733339703889; duration=0sec 2024-12-04T19:15:04,745 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:15:04,745 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6c5c1603a070441ac38abd9294c86a68:C 2024-12-04T19:15:04,751 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-04T19:15:04,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. 2024-12-04T19:15:04,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. as already flushing 2024-12-04T19:15:04,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. 2024-12-04T19:15:04,751 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 6c5c1603a070441ac38abd9294c86a68, NAME => 'TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:04,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 6c5c1603a070441ac38abd9294c86a68, NAME => 'TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:04,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c5c1603a070441ac38abd9294c86a68, NAME => 'TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c5c1603a070441ac38abd9294c86a68, NAME => 'TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:04,758 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/C/53e3c78d791a4358951dd2e96f5133a3 2024-12-04T19:15:04,762 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/A/21ea5931056f45d08b8348e5c21f67dd as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/21ea5931056f45d08b8348e5c21f67dd 2024-12-04T19:15:04,766 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/21ea5931056f45d08b8348e5c21f67dd, entries=150, sequenceid=103, filesize=11.7 K 2024-12-04T19:15:04,767 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/B/e62474a74061431295169a9ec26831c5 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/e62474a74061431295169a9ec26831c5 2024-12-04T19:15:04,770 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/e62474a74061431295169a9ec26831c5, entries=150, sequenceid=103, filesize=11.7 K 2024-12-04T19:15:04,770 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/C/53e3c78d791a4358951dd2e96f5133a3 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/53e3c78d791a4358951dd2e96f5133a3 2024-12-04T19:15:04,774 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/53e3c78d791a4358951dd2e96f5133a3, entries=150, sequenceid=103, filesize=11.7 K 2024-12-04T19:15:04,774 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.03 KB/108576, currentSize=147.60 KB/151140 for 6c5c1603a070441ac38abd9294c86a68 in 859ms, sequenceid=103, compaction requested=false 2024-12-04T19:15:04,774 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6c5c1603a070441ac38abd9294c86a68: 2024-12-04T19:15:04,903 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-04T19:15:04,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. 2024-12-04T19:15:04,904 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing 6c5c1603a070441ac38abd9294c86a68 3/3 column families, dataSize=147.60 KB heapSize=406.08 KB 2024-12-04T19:15:04,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c5c1603a070441ac38abd9294c86a68, store=A 2024-12-04T19:15:04,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T19:15:04,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c5c1603a070441ac38abd9294c86a68, store=B 2024-12-04T19:15:04,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T19:15:04,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c5c1603a070441ac38abd9294c86a68, store=C 2024-12-04T19:15:04,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T19:15:04,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/A/95e105114fcd4d758e6c13c6c2de5480 is 50, key is test_row_0/A:col10/1733339703951/Put/seqid=0 2024-12-04T19:15:04,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742104_1280 (size=12151) 2024-12-04T19:15:05,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(8855): Flush requested on 6c5c1603a070441ac38abd9294c86a68 2024-12-04T19:15:05,113 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. as already flushing 2024-12-04T19:15:05,124 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:05,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37212 deadline: 1733339765121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:05,125 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:05,125 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:05,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37228 deadline: 1733339765122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:05,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:05,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37220 deadline: 1733339765122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:05,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:05,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:05,125 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:05,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37176 deadline: 1733339765122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:05,125 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:05,125 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:05,125 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:05,125 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:05,125 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:05,125 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:05,125 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:05,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37250 deadline: 1733339765122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:05,125 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:05,125 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:05,125 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:05,125 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:05,125 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:05,125 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:05,233 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:05,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37212 deadline: 1733339765232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:05,233 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:05,233 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:05,233 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:05,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37220 deadline: 1733339765232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:05,233 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:05,233 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:05,233 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:05,233 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:05,312 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=141 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/A/95e105114fcd4d758e6c13c6c2de5480 2024-12-04T19:15:05,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/B/751b6a70966f4bd889ddbe6bd9525b5f is 50, key is test_row_0/B:col10/1733339703951/Put/seqid=0 2024-12-04T19:15:05,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742105_1281 (size=12151) 2024-12-04T19:15:05,326 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=141 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/B/751b6a70966f4bd889ddbe6bd9525b5f 2024-12-04T19:15:05,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/C/53175274a1614f489d50be4868316c5d is 50, key is test_row_0/C:col10/1733339703951/Put/seqid=0 2024-12-04T19:15:05,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742106_1282 (size=12151) 2024-12-04T19:15:05,445 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:05,446 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:05,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37220 deadline: 1733339765442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:05,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37212 deadline: 1733339765442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:05,446 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:05,446 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:05,446 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:05,446 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:05,446 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:05,446 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:05,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-04T19:15:05,741 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=141 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/C/53175274a1614f489d50be4868316c5d 2024-12-04T19:15:05,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/A/95e105114fcd4d758e6c13c6c2de5480 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/95e105114fcd4d758e6c13c6c2de5480 2024-12-04T19:15:05,749 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/95e105114fcd4d758e6c13c6c2de5480, entries=150, sequenceid=141, filesize=11.9 K 2024-12-04T19:15:05,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/B/751b6a70966f4bd889ddbe6bd9525b5f as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/751b6a70966f4bd889ddbe6bd9525b5f 2024-12-04T19:15:05,753 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/751b6a70966f4bd889ddbe6bd9525b5f, entries=150, sequenceid=141, filesize=11.9 K 2024-12-04T19:15:05,754 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:05,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37212 deadline: 1733339765751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:05,754 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:05,754 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:05,754 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:05,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37220 deadline: 1733339765752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:05,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/C/53175274a1614f489d50be4868316c5d as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/53175274a1614f489d50be4868316c5d 2024-12-04T19:15:05,754 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:05,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:05,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:05,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:05,759 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/53175274a1614f489d50be4868316c5d, entries=150, sequenceid=141, filesize=11.9 K 2024-12-04T19:15:05,760 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~406.03 KB/415776, currentSize=40.25 KB/41220 for 6c5c1603a070441ac38abd9294c86a68 in 856ms, sequenceid=141, compaction requested=true 2024-12-04T19:15:05,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for 6c5c1603a070441ac38abd9294c86a68: 2024-12-04T19:15:05,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. 2024-12-04T19:15:05,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-12-04T19:15:05,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-12-04T19:15:05,764 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-12-04T19:15:05,764 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2480 sec 2024-12-04T19:15:05,765 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 2.2520 sec 2024-12-04T19:15:06,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(8855): Flush requested on 6c5c1603a070441ac38abd9294c86a68 2024-12-04T19:15:06,134 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6c5c1603a070441ac38abd9294c86a68 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-04T19:15:06,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c5c1603a070441ac38abd9294c86a68, store=A 2024-12-04T19:15:06,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:15:06,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c5c1603a070441ac38abd9294c86a68, store=B 2024-12-04T19:15:06,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:15:06,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c5c1603a070441ac38abd9294c86a68, store=C 2024-12-04T19:15:06,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:15:06,138 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/A/b90855e589394347808aea074cc66c2f is 50, key is test_row_0/A:col10/1733339706133/Put/seqid=0 2024-12-04T19:15:06,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742107_1283 (size=14541) 2024-12-04T19:15:06,167 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733339706159, max timestamp=1733339706165Num uniques 859; 2024-12-04T19:15:06,168 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733339706159, max timestamp=1733339706165Num uniques 859; 2024-12-04T19:15:06,168 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T19:15:06,168 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T19:15:06,167 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733339706159, max timestamp=1733339706165Num uniques 859; 2024-12-04T19:15:06,168 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T19:15:06,196 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:06,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37250 deadline: 1733339766192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:06,197 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:06,197 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:06,197 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:06,200 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:06,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37176 deadline: 1733339766196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:06,201 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:06,201 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:06,201 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:06,201 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:06,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37228 deadline: 1733339766197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:06,201 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:06,202 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:06,202 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:06,266 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:06,266 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:06,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37212 deadline: 1733339766262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:06,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37220 deadline: 1733339766262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:06,266 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:06,266 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:06,266 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:06,266 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:06,266 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:06,266 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:06,304 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:06,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37250 deadline: 1733339766301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:06,304 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:06,304 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:06,304 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:06,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:06,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37176 deadline: 1733339766311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:06,315 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:06,315 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:06,315 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:06,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:06,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37228 deadline: 1733339766312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:06,316 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:06,316 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:06,316 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:06,512 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:06,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37250 deadline: 1733339766511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:06,513 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:06,513 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:06,513 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:06,523 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:06,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37176 deadline: 1733339766521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:06,523 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:06,524 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:06,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37228 deadline: 1733339766521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:06,524 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:06,524 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:06,524 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:06,524 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:06,524 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:06,542 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/A/b90855e589394347808aea074cc66c2f 2024-12-04T19:15:06,547 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/B/cad11e7bfe084c49821b94efbb8f45cc is 50, key is test_row_0/B:col10/1733339706133/Put/seqid=0 2024-12-04T19:15:06,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742108_1284 (size=12151) 2024-12-04T19:15:06,823 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:06,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37250 deadline: 1733339766822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:06,823 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:06,823 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:06,823 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:06,834 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:06,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37176 deadline: 1733339766832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:06,834 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:06,834 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:06,834 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:06,834 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:06,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37228 deadline: 1733339766832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:06,834 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:06,834 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:06,834 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:06,951 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/B/cad11e7bfe084c49821b94efbb8f45cc 2024-12-04T19:15:06,960 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/C/e1c2b2983008489194812aa57aa006f1 is 50, key is test_row_0/C:col10/1733339706133/Put/seqid=0 2024-12-04T19:15:06,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742109_1285 (size=12151) 2024-12-04T19:15:07,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:07,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37220 deadline: 1733339767272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:07,274 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:07,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37212 deadline: 1733339767272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:07,274 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 9ba2734cd39f,33969,1733339629791 for 'test_row_0' in 6c5c1603a070441ac38abd9294c86a68 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2152 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-04T19:15:07,274 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:07,274 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:07,274 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:07,274 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 9ba2734cd39f,33969,1733339629791 for 'test_row_1' in 6c5c1603a070441ac38abd9294c86a68 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2152 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-04T19:15:07,274 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:07,274 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:07,275 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:07,336 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:07,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37250 deadline: 1733339767331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:07,336 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:07,336 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:07,336 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:07,345 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:07,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37176 deadline: 1733339767342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:07,346 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:07,346 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:07,346 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:07,346 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:07,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37228 deadline: 1733339767342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:07,346 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:07,346 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:07,346 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:07,364 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/C/e1c2b2983008489194812aa57aa006f1 2024-12-04T19:15:07,368 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/A/b90855e589394347808aea074cc66c2f as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/b90855e589394347808aea074cc66c2f 2024-12-04T19:15:07,372 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/b90855e589394347808aea074cc66c2f, entries=200, sequenceid=151, filesize=14.2 K 2024-12-04T19:15:07,373 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/B/cad11e7bfe084c49821b94efbb8f45cc as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/cad11e7bfe084c49821b94efbb8f45cc 2024-12-04T19:15:07,376 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/cad11e7bfe084c49821b94efbb8f45cc, entries=150, sequenceid=151, filesize=11.9 K 2024-12-04T19:15:07,377 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/C/e1c2b2983008489194812aa57aa006f1 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/e1c2b2983008489194812aa57aa006f1 2024-12-04T19:15:07,380 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/e1c2b2983008489194812aa57aa006f1, entries=150, sequenceid=151, filesize=11.9 K 2024-12-04T19:15:07,381 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 6c5c1603a070441ac38abd9294c86a68 in 1247ms, sequenceid=151, compaction requested=true 2024-12-04T19:15:07,381 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6c5c1603a070441ac38abd9294c86a68: 2024-12-04T19:15:07,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6c5c1603a070441ac38abd9294c86a68:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T19:15:07,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:15:07,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6c5c1603a070441ac38abd9294c86a68:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T19:15:07,381 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T19:15:07,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:15:07,381 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T19:15:07,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6c5c1603a070441ac38abd9294c86a68:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T19:15:07,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T19:15:07,382 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48510 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T19:15:07,382 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50900 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T19:15:07,382 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1541): 6c5c1603a070441ac38abd9294c86a68/B is initiating minor compaction (all files) 2024-12-04T19:15:07,382 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1541): 6c5c1603a070441ac38abd9294c86a68/A is initiating minor compaction (all files) 2024-12-04T19:15:07,382 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6c5c1603a070441ac38abd9294c86a68/B in TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. 2024-12-04T19:15:07,382 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6c5c1603a070441ac38abd9294c86a68/A in TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. 2024-12-04T19:15:07,382 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/eacc9abf7b5d407988a54784987a450d, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/e62474a74061431295169a9ec26831c5, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/751b6a70966f4bd889ddbe6bd9525b5f, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/cad11e7bfe084c49821b94efbb8f45cc] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp, totalSize=47.4 K 2024-12-04T19:15:07,382 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/27d763afc46340a5a5d4f541d357e7a0, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/21ea5931056f45d08b8348e5c21f67dd, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/95e105114fcd4d758e6c13c6c2de5480, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/b90855e589394347808aea074cc66c2f] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp, totalSize=49.7 K 2024-12-04T19:15:07,382 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting eacc9abf7b5d407988a54784987a450d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1733339703224 2024-12-04T19:15:07,382 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 27d763afc46340a5a5d4f541d357e7a0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1733339703224 2024-12-04T19:15:07,383 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting e62474a74061431295169a9ec26831c5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1733339703264 2024-12-04T19:15:07,383 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 21ea5931056f45d08b8348e5c21f67dd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1733339703264 2024-12-04T19:15:07,388 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 751b6a70966f4bd889ddbe6bd9525b5f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1733339703951 2024-12-04T19:15:07,388 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 95e105114fcd4d758e6c13c6c2de5480, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1733339703951 2024-12-04T19:15:07,388 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting cad11e7bfe084c49821b94efbb8f45cc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1733339705113 2024-12-04T19:15:07,388 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting b90855e589394347808aea074cc66c2f, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1733339705113 2024-12-04T19:15:07,397 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6c5c1603a070441ac38abd9294c86a68#B#compaction#241 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:15:07,397 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6c5c1603a070441ac38abd9294c86a68#A#compaction#240 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:15:07,397 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/B/f984cbfc576d405b89ef6e583d849779 is 50, key is test_row_0/B:col10/1733339706133/Put/seqid=0 2024-12-04T19:15:07,397 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/A/17a6d85075974b8e8a5b51fe16ab9c40 is 50, key is test_row_0/A:col10/1733339706133/Put/seqid=0 2024-12-04T19:15:07,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742110_1286 (size=12493) 2024-12-04T19:15:07,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742111_1287 (size=12493) 2024-12-04T19:15:07,404 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/A/17a6d85075974b8e8a5b51fe16ab9c40 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/17a6d85075974b8e8a5b51fe16ab9c40 2024-12-04T19:15:07,409 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6c5c1603a070441ac38abd9294c86a68/A of 6c5c1603a070441ac38abd9294c86a68 into 17a6d85075974b8e8a5b51fe16ab9c40(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:15:07,409 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6c5c1603a070441ac38abd9294c86a68: 2024-12-04T19:15:07,409 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., storeName=6c5c1603a070441ac38abd9294c86a68/A, priority=12, startTime=1733339707381; duration=0sec 2024-12-04T19:15:07,409 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T19:15:07,409 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6c5c1603a070441ac38abd9294c86a68:A 2024-12-04T19:15:07,409 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T19:15:07,410 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48510 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T19:15:07,410 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1541): 6c5c1603a070441ac38abd9294c86a68/C is initiating minor compaction (all files) 2024-12-04T19:15:07,411 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6c5c1603a070441ac38abd9294c86a68/C in TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. 2024-12-04T19:15:07,411 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/f126694142444ca3a70c7dc9e9890a63, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/53e3c78d791a4358951dd2e96f5133a3, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/53175274a1614f489d50be4868316c5d, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/e1c2b2983008489194812aa57aa006f1] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp, totalSize=47.4 K 2024-12-04T19:15:07,411 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting f126694142444ca3a70c7dc9e9890a63, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1733339703224 2024-12-04T19:15:07,411 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 53e3c78d791a4358951dd2e96f5133a3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1733339703264 2024-12-04T19:15:07,412 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 53175274a1614f489d50be4868316c5d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1733339703951 2024-12-04T19:15:07,412 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting e1c2b2983008489194812aa57aa006f1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1733339705113 2024-12-04T19:15:07,420 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6c5c1603a070441ac38abd9294c86a68#C#compaction#242 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:15:07,420 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/C/53ad0a4dc25b41d4b63ea31c77a7abde is 50, key is test_row_0/C:col10/1733339706133/Put/seqid=0 2024-12-04T19:15:07,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742112_1288 (size=12493) 2024-12-04T19:15:07,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-04T19:15:07,671 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T19:15:07,672 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T19:15:07,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-12-04T19:15:07,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-04T19:15:07,673 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T19:15:07,674 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T19:15:07,674 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T19:15:07,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-04T19:15:07,810 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/B/f984cbfc576d405b89ef6e583d849779 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/f984cbfc576d405b89ef6e583d849779 2024-12-04T19:15:07,814 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6c5c1603a070441ac38abd9294c86a68/B of 6c5c1603a070441ac38abd9294c86a68 into f984cbfc576d405b89ef6e583d849779(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:15:07,814 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6c5c1603a070441ac38abd9294c86a68: 2024-12-04T19:15:07,814 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., storeName=6c5c1603a070441ac38abd9294c86a68/B, priority=12, startTime=1733339707381; duration=0sec 2024-12-04T19:15:07,814 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:15:07,814 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6c5c1603a070441ac38abd9294c86a68:B 2024-12-04T19:15:07,826 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-04T19:15:07,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. 2024-12-04T19:15:07,826 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing 6c5c1603a070441ac38abd9294c86a68 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-12-04T19:15:07,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c5c1603a070441ac38abd9294c86a68, store=A 2024-12-04T19:15:07,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T19:15:07,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c5c1603a070441ac38abd9294c86a68, store=B 2024-12-04T19:15:07,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T19:15:07,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c5c1603a070441ac38abd9294c86a68, store=C 2024-12-04T19:15:07,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T19:15:07,827 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/C/53ad0a4dc25b41d4b63ea31c77a7abde as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/53ad0a4dc25b41d4b63ea31c77a7abde 2024-12-04T19:15:07,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/A/14fa4989bbdb4ccc9b067726dac45d6f is 50, key is test_row_0/A:col10/1733339706194/Put/seqid=0 2024-12-04T19:15:07,835 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6c5c1603a070441ac38abd9294c86a68/C of 6c5c1603a070441ac38abd9294c86a68 into 53ad0a4dc25b41d4b63ea31c77a7abde(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:15:07,835 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6c5c1603a070441ac38abd9294c86a68: 2024-12-04T19:15:07,835 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., storeName=6c5c1603a070441ac38abd9294c86a68/C, priority=12, startTime=1733339707381; duration=0sec 2024-12-04T19:15:07,835 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:15:07,835 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6c5c1603a070441ac38abd9294c86a68:C 2024-12-04T19:15:07,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742113_1289 (size=12151) 2024-12-04T19:15:07,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-04T19:15:08,247 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/A/14fa4989bbdb4ccc9b067726dac45d6f 2024-12-04T19:15:08,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/B/f1798900b50a4327bad01ac424f9b357 is 50, key is test_row_0/B:col10/1733339706194/Put/seqid=0 2024-12-04T19:15:08,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742114_1290 (size=12151) 2024-12-04T19:15:08,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-04T19:15:08,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(8855): Flush requested on 6c5c1603a070441ac38abd9294c86a68 2024-12-04T19:15:08,355 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. as already flushing 2024-12-04T19:15:08,366 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:08,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37250 deadline: 1733339768363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:08,366 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:08,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37228 deadline: 1733339768363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:08,367 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:08,367 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:08,367 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:08,367 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:08,367 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:08,367 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:08,367 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:08,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37176 deadline: 1733339768364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:08,367 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:08,367 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:08,367 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:08,474 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:08,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37250 deadline: 1733339768471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:08,475 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:08,475 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:08,475 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:08,475 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:08,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37228 deadline: 1733339768472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:08,475 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:08,475 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:08,475 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:08,475 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:08,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37176 deadline: 1733339768472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:08,475 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:08,475 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:08,475 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:08,659 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/B/f1798900b50a4327bad01ac424f9b357 2024-12-04T19:15:08,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/C/9aab86884c584f3680fd09039b58808a is 50, key is test_row_0/C:col10/1733339706194/Put/seqid=0 2024-12-04T19:15:08,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742115_1291 (size=12151) 2024-12-04T19:15:08,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:08,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37250 deadline: 1733339768681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:08,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:08,684 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:08,684 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:08,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37228 deadline: 1733339768682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:08,684 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:08,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37176 deadline: 1733339768682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:08,684 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:08,685 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:08,685 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:08,685 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:08,685 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:08,685 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:08,685 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:08,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-04T19:15:08,995 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:08,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37250 deadline: 1733339768992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:08,996 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:08,996 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:08,996 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:08,996 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:08,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37228 deadline: 1733339768992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:08,996 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:08,996 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:08,996 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:08,996 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:08,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37176 deadline: 1733339768992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:08,996 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:08,996 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:08,996 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:09,068 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/C/9aab86884c584f3680fd09039b58808a 2024-12-04T19:15:09,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/A/14fa4989bbdb4ccc9b067726dac45d6f as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/14fa4989bbdb4ccc9b067726dac45d6f 2024-12-04T19:15:09,076 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/14fa4989bbdb4ccc9b067726dac45d6f, entries=150, sequenceid=189, filesize=11.9 K 2024-12-04T19:15:09,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/B/f1798900b50a4327bad01ac424f9b357 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/f1798900b50a4327bad01ac424f9b357 2024-12-04T19:15:09,081 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/f1798900b50a4327bad01ac424f9b357, entries=150, sequenceid=189, filesize=11.9 K 2024-12-04T19:15:09,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/C/9aab86884c584f3680fd09039b58808a as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/9aab86884c584f3680fd09039b58808a 2024-12-04T19:15:09,086 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/9aab86884c584f3680fd09039b58808a, entries=150, sequenceid=189, filesize=11.9 K 2024-12-04T19:15:09,087 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 6c5c1603a070441ac38abd9294c86a68 in 1260ms, sequenceid=189, compaction requested=false 2024-12-04T19:15:09,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for 6c5c1603a070441ac38abd9294c86a68: 2024-12-04T19:15:09,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. 2024-12-04T19:15:09,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-12-04T19:15:09,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-12-04T19:15:09,089 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-12-04T19:15:09,089 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4140 sec 2024-12-04T19:15:09,091 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 1.4180 sec 2024-12-04T19:15:09,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(8855): Flush requested on 6c5c1603a070441ac38abd9294c86a68 2024-12-04T19:15:09,293 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6c5c1603a070441ac38abd9294c86a68 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-04T19:15:09,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c5c1603a070441ac38abd9294c86a68, store=A 2024-12-04T19:15:09,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:15:09,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c5c1603a070441ac38abd9294c86a68, store=B 2024-12-04T19:15:09,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:15:09,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c5c1603a070441ac38abd9294c86a68, store=C 2024-12-04T19:15:09,295 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:15:09,298 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/A/c1a884816070474cbd3a878ff3a6a32e is 50, key is test_row_0/A:col10/1733339709292/Put/seqid=0 2024-12-04T19:15:09,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742116_1292 (size=14541) 2024-12-04T19:15:09,327 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:09,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37212 deadline: 1733339769324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:09,327 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:09,327 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:09,328 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:09,329 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:09,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37220 deadline: 1733339769327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:09,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:09,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:09,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:09,433 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:09,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37212 deadline: 1733339769431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:09,433 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:09,434 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:09,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37220 deadline: 1733339769432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:09,434 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:09,434 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:09,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:09,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:09,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:09,502 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:09,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37250 deadline: 1733339769501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:09,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:09,503 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:09,503 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:09,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37228 deadline: 1733339769501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:09,503 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:09,503 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:09,503 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:09,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:09,503 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:09,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37176 deadline: 1733339769501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:09,503 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:09,503 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:09,503 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:09,644 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:09,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37212 deadline: 1733339769641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:09,644 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:09,644 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:09,644 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:09,644 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:09,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37220 deadline: 1733339769642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:09,645 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:09,645 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:09,645 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:09,702 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/A/c1a884816070474cbd3a878ff3a6a32e 2024-12-04T19:15:09,709 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/B/233f006db5e34bcbb7cbd5da64df6b34 is 50, key is test_row_0/B:col10/1733339709292/Put/seqid=0 2024-12-04T19:15:09,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742117_1293 (size=12151) 2024-12-04T19:15:09,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-04T19:15:09,822 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T19:15:09,823 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T19:15:09,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-12-04T19:15:09,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-04T19:15:09,824 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T19:15:09,824 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T19:15:09,825 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T19:15:09,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-04T19:15:09,954 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:09,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37212 deadline: 1733339769952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:09,954 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:09,954 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:09,954 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:09,954 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:09,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37220 deadline: 1733339769952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:09,955 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:09,955 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:09,955 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:09,977 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-04T19:15:09,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. 2024-12-04T19:15:09,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. as already flushing 2024-12-04T19:15:09,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. 2024-12-04T19:15:09,977 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 6c5c1603a070441ac38abd9294c86a68, NAME => 'TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:09,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 6c5c1603a070441ac38abd9294c86a68, NAME => 'TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:09,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c5c1603a070441ac38abd9294c86a68, NAME => 'TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c5c1603a070441ac38abd9294c86a68, NAME => 'TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:10,113 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/B/233f006db5e34bcbb7cbd5da64df6b34 2024-12-04T19:15:10,118 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/C/a8d9868e17bb4debb8216351bd3b519b is 50, key is test_row_0/C:col10/1733339709292/Put/seqid=0 2024-12-04T19:15:10,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742118_1294 (size=12151) 2024-12-04T19:15:10,129 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-04T19:15:10,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. 2024-12-04T19:15:10,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. as already flushing 2024-12-04T19:15:10,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. 2024-12-04T19:15:10,130 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 6c5c1603a070441ac38abd9294c86a68, NAME => 'TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:10,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 6c5c1603a070441ac38abd9294c86a68, NAME => 'TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:10,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c5c1603a070441ac38abd9294c86a68, NAME => 'TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c5c1603a070441ac38abd9294c86a68, NAME => 'TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:10,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-04T19:15:10,282 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-04T19:15:10,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. 2024-12-04T19:15:10,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. as already flushing 2024-12-04T19:15:10,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. 2024-12-04T19:15:10,282 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 6c5c1603a070441ac38abd9294c86a68, NAME => 'TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:10,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 6c5c1603a070441ac38abd9294c86a68, NAME => 'TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:10,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c5c1603a070441ac38abd9294c86a68, NAME => 'TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c5c1603a070441ac38abd9294c86a68, NAME => 'TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:10,435 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-04T19:15:10,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. 2024-12-04T19:15:10,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. as already flushing 2024-12-04T19:15:10,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. 2024-12-04T19:15:10,435 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 6c5c1603a070441ac38abd9294c86a68, NAME => 'TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:10,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 6c5c1603a070441ac38abd9294c86a68, NAME => 'TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:10,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c5c1603a070441ac38abd9294c86a68, NAME => 'TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c5c1603a070441ac38abd9294c86a68, NAME => 'TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:10,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-04T19:15:10,463 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:10,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37212 deadline: 1733339770462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:10,464 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:10,464 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:10,464 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:10,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37220 deadline: 1733339770462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:10,464 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:10,464 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:10,464 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:10,464 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:10,522 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/C/a8d9868e17bb4debb8216351bd3b519b 2024-12-04T19:15:10,524 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:10,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37250 deadline: 1733339770522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:10,524 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:10,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37228 deadline: 1733339770522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:10,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:10,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37176 deadline: 1733339770522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:10,525 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 9ba2734cd39f,33969,1733339629791 for 'test_row_0' in 6c5c1603a070441ac38abd9294c86a68 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-04T19:15:10,525 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:10,525 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:10,525 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 9ba2734cd39f,33969,1733339629791 for 'test_row_2' in 6c5c1603a070441ac38abd9294c86a68 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-04T19:15:10,525 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:10,525 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:10,525 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:10,525 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:10,525 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 9ba2734cd39f,33969,1733339629791 for 'test_row_2' in 6c5c1603a070441ac38abd9294c86a68 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-04T19:15:10,526 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:10,526 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:10,526 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:10,526 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/A/c1a884816070474cbd3a878ff3a6a32e as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/c1a884816070474cbd3a878ff3a6a32e 2024-12-04T19:15:10,529 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/c1a884816070474cbd3a878ff3a6a32e, entries=200, sequenceid=202, filesize=14.2 K 2024-12-04T19:15:10,530 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/B/233f006db5e34bcbb7cbd5da64df6b34 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/233f006db5e34bcbb7cbd5da64df6b34 2024-12-04T19:15:10,534 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/233f006db5e34bcbb7cbd5da64df6b34, entries=150, sequenceid=202, filesize=11.9 K 2024-12-04T19:15:10,534 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/C/a8d9868e17bb4debb8216351bd3b519b as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/a8d9868e17bb4debb8216351bd3b519b 2024-12-04T19:15:10,538 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/a8d9868e17bb4debb8216351bd3b519b, entries=150, sequenceid=202, filesize=11.9 K 2024-12-04T19:15:10,538 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 6c5c1603a070441ac38abd9294c86a68 in 1245ms, sequenceid=202, compaction requested=true 2024-12-04T19:15:10,538 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6c5c1603a070441ac38abd9294c86a68: 2024-12-04T19:15:10,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6c5c1603a070441ac38abd9294c86a68:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T19:15:10,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:15:10,539 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T19:15:10,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6c5c1603a070441ac38abd9294c86a68:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T19:15:10,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:15:10,539 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T19:15:10,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6c5c1603a070441ac38abd9294c86a68:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T19:15:10,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T19:15:10,540 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39185 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T19:15:10,540 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T19:15:10,540 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1541): 6c5c1603a070441ac38abd9294c86a68/A is initiating minor compaction (all files) 2024-12-04T19:15:10,540 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1541): 6c5c1603a070441ac38abd9294c86a68/B is initiating minor compaction (all files) 2024-12-04T19:15:10,540 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6c5c1603a070441ac38abd9294c86a68/A in TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. 2024-12-04T19:15:10,540 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6c5c1603a070441ac38abd9294c86a68/B in TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. 2024-12-04T19:15:10,540 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/17a6d85075974b8e8a5b51fe16ab9c40, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/14fa4989bbdb4ccc9b067726dac45d6f, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/c1a884816070474cbd3a878ff3a6a32e] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp, totalSize=38.3 K 2024-12-04T19:15:10,540 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/f984cbfc576d405b89ef6e583d849779, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/f1798900b50a4327bad01ac424f9b357, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/233f006db5e34bcbb7cbd5da64df6b34] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp, totalSize=35.9 K 2024-12-04T19:15:10,540 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 17a6d85075974b8e8a5b51fe16ab9c40, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1733339705113 2024-12-04T19:15:10,540 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting f984cbfc576d405b89ef6e583d849779, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1733339705113 2024-12-04T19:15:10,540 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 14fa4989bbdb4ccc9b067726dac45d6f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1733339706191 2024-12-04T19:15:10,541 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting f1798900b50a4327bad01ac424f9b357, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1733339706191 2024-12-04T19:15:10,541 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting c1a884816070474cbd3a878ff3a6a32e, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1733339708361 2024-12-04T19:15:10,541 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 233f006db5e34bcbb7cbd5da64df6b34, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1733339708362 2024-12-04T19:15:10,547 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6c5c1603a070441ac38abd9294c86a68#B#compaction#249 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:15:10,547 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/B/0fcd19f034784ea1a4df54c21e6b5c44 is 50, key is test_row_0/B:col10/1733339709292/Put/seqid=0 2024-12-04T19:15:10,549 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6c5c1603a070441ac38abd9294c86a68#A#compaction#250 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:15:10,550 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/A/fb281ae36a1949a1952ac2b5f45f7ea7 is 50, key is test_row_0/A:col10/1733339709292/Put/seqid=0 2024-12-04T19:15:10,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742119_1295 (size=12595) 2024-12-04T19:15:10,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742120_1296 (size=12595) 2024-12-04T19:15:10,587 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-04T19:15:10,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. 2024-12-04T19:15:10,587 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing 6c5c1603a070441ac38abd9294c86a68 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-04T19:15:10,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c5c1603a070441ac38abd9294c86a68, store=A 2024-12-04T19:15:10,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:15:10,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c5c1603a070441ac38abd9294c86a68, store=B 2024-12-04T19:15:10,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:15:10,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c5c1603a070441ac38abd9294c86a68, store=C 2024-12-04T19:15:10,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:15:10,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/A/4488084553434cedb90010ec357e5a13 is 50, key is test_row_0/A:col10/1733339709321/Put/seqid=0 2024-12-04T19:15:10,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742121_1297 (size=12151) 2024-12-04T19:15:10,958 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/B/0fcd19f034784ea1a4df54c21e6b5c44 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/0fcd19f034784ea1a4df54c21e6b5c44 2024-12-04T19:15:10,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-04T19:15:10,962 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6c5c1603a070441ac38abd9294c86a68/B of 6c5c1603a070441ac38abd9294c86a68 into 0fcd19f034784ea1a4df54c21e6b5c44(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:15:10,962 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6c5c1603a070441ac38abd9294c86a68: 2024-12-04T19:15:10,962 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., storeName=6c5c1603a070441ac38abd9294c86a68/B, priority=13, startTime=1733339710539; duration=0sec 2024-12-04T19:15:10,962 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T19:15:10,962 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6c5c1603a070441ac38abd9294c86a68:B 2024-12-04T19:15:10,962 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T19:15:10,963 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T19:15:10,963 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1541): 6c5c1603a070441ac38abd9294c86a68/C is initiating minor compaction (all files) 2024-12-04T19:15:10,963 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6c5c1603a070441ac38abd9294c86a68/C in TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. 2024-12-04T19:15:10,963 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/53ad0a4dc25b41d4b63ea31c77a7abde, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/9aab86884c584f3680fd09039b58808a, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/a8d9868e17bb4debb8216351bd3b519b] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp, totalSize=35.9 K 2024-12-04T19:15:10,963 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 53ad0a4dc25b41d4b63ea31c77a7abde, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1733339705113 2024-12-04T19:15:10,964 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 9aab86884c584f3680fd09039b58808a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1733339706191 2024-12-04T19:15:10,964 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting a8d9868e17bb4debb8216351bd3b519b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1733339708362 2024-12-04T19:15:10,966 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/A/fb281ae36a1949a1952ac2b5f45f7ea7 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/fb281ae36a1949a1952ac2b5f45f7ea7 2024-12-04T19:15:10,970 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6c5c1603a070441ac38abd9294c86a68/A of 6c5c1603a070441ac38abd9294c86a68 into fb281ae36a1949a1952ac2b5f45f7ea7(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:15:10,970 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6c5c1603a070441ac38abd9294c86a68: 2024-12-04T19:15:10,970 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., storeName=6c5c1603a070441ac38abd9294c86a68/A, priority=13, startTime=1733339710539; duration=0sec 2024-12-04T19:15:10,970 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:15:10,970 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6c5c1603a070441ac38abd9294c86a68:A 2024-12-04T19:15:10,970 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6c5c1603a070441ac38abd9294c86a68#C#compaction#252 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:15:10,971 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/C/f86f0e4d90254cb1a88b430f5fa44385 is 50, key is test_row_0/C:col10/1733339709292/Put/seqid=0 2024-12-04T19:15:10,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742122_1298 (size=12595) 2024-12-04T19:15:10,995 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=222 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/A/4488084553434cedb90010ec357e5a13 2024-12-04T19:15:11,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/B/b025ed1d183b470aa1c50a7db15771fc is 50, key is test_row_0/B:col10/1733339709321/Put/seqid=0 2024-12-04T19:15:11,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742123_1299 (size=12151) 2024-12-04T19:15:11,393 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/C/f86f0e4d90254cb1a88b430f5fa44385 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/f86f0e4d90254cb1a88b430f5fa44385 2024-12-04T19:15:11,397 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6c5c1603a070441ac38abd9294c86a68/C of 6c5c1603a070441ac38abd9294c86a68 into f86f0e4d90254cb1a88b430f5fa44385(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:15:11,397 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6c5c1603a070441ac38abd9294c86a68: 2024-12-04T19:15:11,397 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., storeName=6c5c1603a070441ac38abd9294c86a68/C, priority=13, startTime=1733339710539; duration=0sec 2024-12-04T19:15:11,397 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:15:11,397 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6c5c1603a070441ac38abd9294c86a68:C 2024-12-04T19:15:11,407 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=222 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/B/b025ed1d183b470aa1c50a7db15771fc 2024-12-04T19:15:11,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/C/0efc09240177462596022f413b802f61 is 50, key is test_row_0/C:col10/1733339709321/Put/seqid=0 2024-12-04T19:15:11,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742124_1300 (size=12151) 2024-12-04T19:15:11,420 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=222 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/C/0efc09240177462596022f413b802f61 2024-12-04T19:15:11,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/A/4488084553434cedb90010ec357e5a13 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/4488084553434cedb90010ec357e5a13 2024-12-04T19:15:11,427 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/4488084553434cedb90010ec357e5a13, entries=150, sequenceid=222, filesize=11.9 K 2024-12-04T19:15:11,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/B/b025ed1d183b470aa1c50a7db15771fc as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/b025ed1d183b470aa1c50a7db15771fc 2024-12-04T19:15:11,431 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/b025ed1d183b470aa1c50a7db15771fc, entries=150, sequenceid=222, filesize=11.9 K 2024-12-04T19:15:11,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/C/0efc09240177462596022f413b802f61 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/0efc09240177462596022f413b802f61 2024-12-04T19:15:11,435 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/0efc09240177462596022f413b802f61, entries=150, sequenceid=222, filesize=11.9 K 2024-12-04T19:15:11,436 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=0 B/0 for 6c5c1603a070441ac38abd9294c86a68 in 849ms, sequenceid=222, compaction requested=false 2024-12-04T19:15:11,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for 6c5c1603a070441ac38abd9294c86a68: 2024-12-04T19:15:11,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. 2024-12-04T19:15:11,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-12-04T19:15:11,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-12-04T19:15:11,439 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-12-04T19:15:11,439 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6120 sec 2024-12-04T19:15:11,441 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 1.6160 sec 2024-12-04T19:15:11,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(8855): Flush requested on 6c5c1603a070441ac38abd9294c86a68 2024-12-04T19:15:11,483 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6c5c1603a070441ac38abd9294c86a68 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-04T19:15:11,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c5c1603a070441ac38abd9294c86a68, store=A 2024-12-04T19:15:11,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:15:11,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c5c1603a070441ac38abd9294c86a68, store=B 2024-12-04T19:15:11,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:15:11,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c5c1603a070441ac38abd9294c86a68, store=C 2024-12-04T19:15:11,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:15:11,487 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/A/a0d776413f1d4bc38705cf93911376aa is 50, key is test_row_0/A:col10/1733339711479/Put/seqid=0 2024-12-04T19:15:11,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742125_1301 (size=12151) 2024-12-04T19:15:11,519 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733339711500, max timestamp=1733339711517Num uniques 859; 2024-12-04T19:15:11,519 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T19:15:11,519 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733339711500, max timestamp=1733339711517Num uniques 859; 2024-12-04T19:15:11,519 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T19:15:11,519 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733339711500, max timestamp=1733339711517Num uniques 859; 2024-12-04T19:15:11,519 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T19:15:11,548 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:11,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37212 deadline: 1733339771545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:11,548 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:11,548 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:11,548 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:11,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:11,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37220 deadline: 1733339771547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:11,551 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:11,551 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:11,551 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:11,653 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:11,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37212 deadline: 1733339771652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:11,653 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:11,653 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:11,653 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:11,664 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:11,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37220 deadline: 1733339771662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:11,665 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:11,665 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:11,665 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:11,864 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:11,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37212 deadline: 1733339771861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:11,864 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:11,864 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:11,864 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:11,874 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:11,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37220 deadline: 1733339771872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:11,874 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:11,874 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:11,874 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:11,891 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/A/a0d776413f1d4bc38705cf93911376aa 2024-12-04T19:15:11,897 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/B/4186a22a6dd1412faa24e75052b69f9b is 50, key is test_row_0/B:col10/1733339711479/Put/seqid=0 2024-12-04T19:15:11,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742126_1302 (size=12151) 2024-12-04T19:15:11,900 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/B/4186a22a6dd1412faa24e75052b69f9b 2024-12-04T19:15:11,906 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/C/0d0b554e65b2422d8943336dc4002365 is 50, key is test_row_0/C:col10/1733339711479/Put/seqid=0 2024-12-04T19:15:11,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742127_1303 (size=12151) 2024-12-04T19:15:11,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-04T19:15:11,982 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T19:15:11,982 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T19:15:11,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-12-04T19:15:11,984 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T19:15:11,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-04T19:15:11,984 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T19:15:11,985 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T19:15:12,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-04T19:15:12,137 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-04T19:15:12,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. 2024-12-04T19:15:12,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. as already flushing 2024-12-04T19:15:12,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. 2024-12-04T19:15:12,137 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 6c5c1603a070441ac38abd9294c86a68, NAME => 'TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:12,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 6c5c1603a070441ac38abd9294c86a68, NAME => 'TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:12,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c5c1603a070441ac38abd9294c86a68, NAME => 'TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c5c1603a070441ac38abd9294c86a68, NAME => 'TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:12,175 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:12,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37212 deadline: 1733339772171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:12,175 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:12,175 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:12,175 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:12,184 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:12,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37220 deadline: 1733339772181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:12,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:12,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:12,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:12,289 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-04T19:15:12,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. 2024-12-04T19:15:12,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. as already flushing 2024-12-04T19:15:12,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. 2024-12-04T19:15:12,290 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 6c5c1603a070441ac38abd9294c86a68, NAME => 'TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:12,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 6c5c1603a070441ac38abd9294c86a68, NAME => 'TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:12,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c5c1603a070441ac38abd9294c86a68, NAME => 'TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c5c1603a070441ac38abd9294c86a68, NAME => 'TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:12,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-04T19:15:12,309 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/C/0d0b554e65b2422d8943336dc4002365 2024-12-04T19:15:12,313 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/A/a0d776413f1d4bc38705cf93911376aa as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/a0d776413f1d4bc38705cf93911376aa 2024-12-04T19:15:12,317 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/a0d776413f1d4bc38705cf93911376aa, entries=150, sequenceid=235, filesize=11.9 K 2024-12-04T19:15:12,318 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/B/4186a22a6dd1412faa24e75052b69f9b as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/4186a22a6dd1412faa24e75052b69f9b 2024-12-04T19:15:12,321 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/4186a22a6dd1412faa24e75052b69f9b, entries=150, sequenceid=235, filesize=11.9 K 2024-12-04T19:15:12,322 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/C/0d0b554e65b2422d8943336dc4002365 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/0d0b554e65b2422d8943336dc4002365 2024-12-04T19:15:12,325 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/0d0b554e65b2422d8943336dc4002365, entries=150, sequenceid=235, filesize=11.9 K 2024-12-04T19:15:12,326 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 6c5c1603a070441ac38abd9294c86a68 in 843ms, sequenceid=235, compaction requested=true 2024-12-04T19:15:12,326 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6c5c1603a070441ac38abd9294c86a68: 2024-12-04T19:15:12,326 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6c5c1603a070441ac38abd9294c86a68:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T19:15:12,326 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:15:12,326 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6c5c1603a070441ac38abd9294c86a68:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T19:15:12,326 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:15:12,326 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T19:15:12,326 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6c5c1603a070441ac38abd9294c86a68:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T19:15:12,326 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T19:15:12,326 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T19:15:12,327 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T19:15:12,327 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1541): 6c5c1603a070441ac38abd9294c86a68/B is initiating minor compaction (all files) 2024-12-04T19:15:12,327 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6c5c1603a070441ac38abd9294c86a68/B in TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. 2024-12-04T19:15:12,327 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/0fcd19f034784ea1a4df54c21e6b5c44, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/b025ed1d183b470aa1c50a7db15771fc, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/4186a22a6dd1412faa24e75052b69f9b] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp, totalSize=36.0 K 2024-12-04T19:15:12,327 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T19:15:12,327 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1541): 6c5c1603a070441ac38abd9294c86a68/A is initiating minor compaction (all files) 2024-12-04T19:15:12,327 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6c5c1603a070441ac38abd9294c86a68/A in TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. 2024-12-04T19:15:12,327 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 0fcd19f034784ea1a4df54c21e6b5c44, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1733339708362 2024-12-04T19:15:12,327 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/fb281ae36a1949a1952ac2b5f45f7ea7, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/4488084553434cedb90010ec357e5a13, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/a0d776413f1d4bc38705cf93911376aa] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp, totalSize=36.0 K 2024-12-04T19:15:12,328 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting b025ed1d183b470aa1c50a7db15771fc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1733339709309 2024-12-04T19:15:12,328 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting fb281ae36a1949a1952ac2b5f45f7ea7, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1733339708362 2024-12-04T19:15:12,328 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 4186a22a6dd1412faa24e75052b69f9b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1733339711473 2024-12-04T19:15:12,328 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4488084553434cedb90010ec357e5a13, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1733339709309 2024-12-04T19:15:12,328 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting a0d776413f1d4bc38705cf93911376aa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1733339711473 2024-12-04T19:15:12,333 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6c5c1603a070441ac38abd9294c86a68#A#compaction#259 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:15:12,333 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6c5c1603a070441ac38abd9294c86a68#B#compaction#258 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:15:12,333 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/B/b5d0de0166b544f7ae1c0dc2b36a0ec1 is 50, key is test_row_0/B:col10/1733339711479/Put/seqid=0 2024-12-04T19:15:12,333 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/A/7f3c4e3a33ce41aeb2b009b13c7f75e0 is 50, key is test_row_0/A:col10/1733339711479/Put/seqid=0 2024-12-04T19:15:12,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742128_1304 (size=12697) 2024-12-04T19:15:12,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742129_1305 (size=12697) 2024-12-04T19:15:12,442 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-04T19:15:12,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. 2024-12-04T19:15:12,442 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing 6c5c1603a070441ac38abd9294c86a68 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-12-04T19:15:12,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c5c1603a070441ac38abd9294c86a68, store=A 2024-12-04T19:15:12,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T19:15:12,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c5c1603a070441ac38abd9294c86a68, store=B 2024-12-04T19:15:12,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T19:15:12,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c5c1603a070441ac38abd9294c86a68, store=C 2024-12-04T19:15:12,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T19:15:12,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/A/872daeb46d904501be56a9dab8911cd8 is 50, key is test_row_0/A:col10/1733339711546/Put/seqid=0 2024-12-04T19:15:12,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742130_1306 (size=12301) 2024-12-04T19:15:12,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(8855): Flush requested on 6c5c1603a070441ac38abd9294c86a68 2024-12-04T19:15:12,536 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. as already flushing 2024-12-04T19:15:12,550 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:12,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37176 deadline: 1733339772544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:12,550 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:12,550 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:12,550 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:12,552 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:12,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37228 deadline: 1733339772549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:12,553 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:12,553 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:12,553 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:12,554 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:12,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37250 deadline: 1733339772550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:12,554 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:12,554 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:12,554 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:12,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-04T19:15:12,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:12,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37176 deadline: 1733339772661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:12,664 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:12,664 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:12,664 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:12,664 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:12,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37228 deadline: 1733339772661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:12,664 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:12,664 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:12,664 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:12,664 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:12,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37250 deadline: 1733339772661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:12,664 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:12,664 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:12,664 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:12,684 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:12,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37212 deadline: 1733339772682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:12,685 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:12,685 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:12,685 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:12,695 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:12,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37220 deadline: 1733339772692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:12,695 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:12,695 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:12,695 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:12,741 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/B/b5d0de0166b544f7ae1c0dc2b36a0ec1 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/b5d0de0166b544f7ae1c0dc2b36a0ec1 2024-12-04T19:15:12,741 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/A/7f3c4e3a33ce41aeb2b009b13c7f75e0 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/7f3c4e3a33ce41aeb2b009b13c7f75e0 2024-12-04T19:15:12,745 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6c5c1603a070441ac38abd9294c86a68/A of 6c5c1603a070441ac38abd9294c86a68 into 7f3c4e3a33ce41aeb2b009b13c7f75e0(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:15:12,745 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6c5c1603a070441ac38abd9294c86a68/B of 6c5c1603a070441ac38abd9294c86a68 into b5d0de0166b544f7ae1c0dc2b36a0ec1(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:15:12,745 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6c5c1603a070441ac38abd9294c86a68: 2024-12-04T19:15:12,745 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6c5c1603a070441ac38abd9294c86a68: 2024-12-04T19:15:12,745 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., storeName=6c5c1603a070441ac38abd9294c86a68/A, priority=13, startTime=1733339712326; duration=0sec 2024-12-04T19:15:12,745 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., storeName=6c5c1603a070441ac38abd9294c86a68/B, priority=13, startTime=1733339712326; duration=0sec 2024-12-04T19:15:12,745 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T19:15:12,745 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6c5c1603a070441ac38abd9294c86a68:A 2024-12-04T19:15:12,745 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T19:15:12,745 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6c5c1603a070441ac38abd9294c86a68:B 2024-12-04T19:15:12,745 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T19:15:12,746 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T19:15:12,746 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1541): 6c5c1603a070441ac38abd9294c86a68/C is initiating minor compaction (all files) 2024-12-04T19:15:12,746 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6c5c1603a070441ac38abd9294c86a68/C in TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. 2024-12-04T19:15:12,746 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/f86f0e4d90254cb1a88b430f5fa44385, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/0efc09240177462596022f413b802f61, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/0d0b554e65b2422d8943336dc4002365] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp, totalSize=36.0 K 2024-12-04T19:15:12,746 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting f86f0e4d90254cb1a88b430f5fa44385, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1733339708362 2024-12-04T19:15:12,747 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0efc09240177462596022f413b802f61, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1733339709309 2024-12-04T19:15:12,747 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0d0b554e65b2422d8943336dc4002365, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1733339711473 2024-12-04T19:15:12,753 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6c5c1603a070441ac38abd9294c86a68#C#compaction#261 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:15:12,754 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/C/8a143da60c644a44b451bef3fa6ad266 is 50, key is test_row_0/C:col10/1733339711479/Put/seqid=0 2024-12-04T19:15:12,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742131_1307 (size=12697) 2024-12-04T19:15:12,850 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/A/872daeb46d904501be56a9dab8911cd8 2024-12-04T19:15:12,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/B/66d74161b2c7444e8ac96c6224524518 is 50, key is test_row_0/B:col10/1733339711546/Put/seqid=0 2024-12-04T19:15:12,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742132_1308 (size=12301) 2024-12-04T19:15:12,874 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:12,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37176 deadline: 1733339772871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:12,874 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:12,874 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:12,874 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:12,874 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:12,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37228 deadline: 1733339772871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:12,875 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:12,875 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:12,875 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:12,875 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:12,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37250 deadline: 1733339772871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:12,875 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:12,875 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:12,875 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:13,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-04T19:15:13,169 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/C/8a143da60c644a44b451bef3fa6ad266 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/8a143da60c644a44b451bef3fa6ad266 2024-12-04T19:15:13,172 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6c5c1603a070441ac38abd9294c86a68/C of 6c5c1603a070441ac38abd9294c86a68 into 8a143da60c644a44b451bef3fa6ad266(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:15:13,172 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6c5c1603a070441ac38abd9294c86a68: 2024-12-04T19:15:13,172 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., storeName=6c5c1603a070441ac38abd9294c86a68/C, priority=13, startTime=1733339712326; duration=0sec 2024-12-04T19:15:13,173 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:15:13,173 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6c5c1603a070441ac38abd9294c86a68:C 2024-12-04T19:15:13,184 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:13,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37176 deadline: 1733339773182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:13,184 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:13,184 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:13,184 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:13,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37228 deadline: 1733339773182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:13,184 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:13,185 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:13,185 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:13,185 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:13,185 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:13,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37250 deadline: 1733339773182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:13,185 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:13,185 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:13,185 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:13,260 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/B/66d74161b2c7444e8ac96c6224524518 2024-12-04T19:15:13,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/C/b0964ab1e88a45cfa5079a7a2d6a683f is 50, key is test_row_0/C:col10/1733339711546/Put/seqid=0 2024-12-04T19:15:13,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742133_1309 (size=12301) 2024-12-04T19:15:13,678 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/C/b0964ab1e88a45cfa5079a7a2d6a683f 2024-12-04T19:15:13,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:13,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37176 deadline: 1733339773691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:13,694 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:13,694 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:13,694 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:13,694 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:13,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37250 deadline: 1733339773691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:13,694 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:13,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37228 deadline: 1733339773691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:13,694 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:13,694 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:13,694 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:13,694 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:13,694 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:13,694 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:13,703 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:13,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37212 deadline: 1733339773702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:13,703 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:13,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37220 deadline: 1733339773702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:13,704 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 9ba2734cd39f,33969,1733339629791 for 'test_row_0' in 6c5c1603a070441ac38abd9294c86a68 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2156 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-04T19:15:13,704 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 9ba2734cd39f,33969,1733339629791 for 'test_row_2' in 6c5c1603a070441ac38abd9294c86a68 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2158 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-04T19:15:13,704 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:13,704 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:13,704 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:13,704 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:13,704 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:13,704 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:13,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/A/872daeb46d904501be56a9dab8911cd8 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/872daeb46d904501be56a9dab8911cd8 2024-12-04T19:15:13,741 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/872daeb46d904501be56a9dab8911cd8, entries=150, sequenceid=271, filesize=12.0 K 2024-12-04T19:15:13,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/B/66d74161b2c7444e8ac96c6224524518 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/66d74161b2c7444e8ac96c6224524518 2024-12-04T19:15:13,745 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/66d74161b2c7444e8ac96c6224524518, entries=150, sequenceid=271, filesize=12.0 K 2024-12-04T19:15:13,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/C/b0964ab1e88a45cfa5079a7a2d6a683f as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/b0964ab1e88a45cfa5079a7a2d6a683f 2024-12-04T19:15:13,748 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/b0964ab1e88a45cfa5079a7a2d6a683f, entries=150, sequenceid=271, filesize=12.0 K 2024-12-04T19:15:13,749 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 6c5c1603a070441ac38abd9294c86a68 in 1307ms, sequenceid=271, compaction requested=false 2024-12-04T19:15:13,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for 6c5c1603a070441ac38abd9294c86a68: 2024-12-04T19:15:13,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. 2024-12-04T19:15:13,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-12-04T19:15:13,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-12-04T19:15:13,753 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-12-04T19:15:13,753 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7670 sec 2024-12-04T19:15:13,755 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 1.7720 sec 2024-12-04T19:15:14,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-04T19:15:14,132 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T19:15:14,133 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T19:15:14,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-12-04T19:15:14,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-04T19:15:14,135 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T19:15:14,136 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T19:15:14,136 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T19:15:14,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-04T19:15:14,301 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-04T19:15:14,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. 2024-12-04T19:15:14,301 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing 6c5c1603a070441ac38abd9294c86a68 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-04T19:15:14,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c5c1603a070441ac38abd9294c86a68, store=A 2024-12-04T19:15:14,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:15:14,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c5c1603a070441ac38abd9294c86a68, store=B 2024-12-04T19:15:14,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:15:14,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c5c1603a070441ac38abd9294c86a68, store=C 2024-12-04T19:15:14,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:15:14,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/A/385292fa7a77471bb165da5bc1223c92 is 50, key is test_row_0/A:col10/1733339712543/Put/seqid=0 2024-12-04T19:15:14,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742134_1310 (size=12301) 2024-12-04T19:15:14,325 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/A/385292fa7a77471bb165da5bc1223c92 2024-12-04T19:15:14,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/B/c7bd652b08ed43c0bcaef7a8dda50fcb is 50, key is test_row_0/B:col10/1733339712543/Put/seqid=0 2024-12-04T19:15:14,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742135_1311 (size=12301) 2024-12-04T19:15:14,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-04T19:15:14,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(8855): Flush requested on 6c5c1603a070441ac38abd9294c86a68 2024-12-04T19:15:14,704 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. as already flushing 2024-12-04T19:15:14,731 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733339714721, max timestamp=1733339714728Num uniques 859; 2024-12-04T19:15:14,731 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-04T19:15:14,731 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733339714721, max timestamp=1733339714728Num uniques 859; 2024-12-04T19:15:14,731 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733339714721, max timestamp=1733339714728Num uniques 859; 2024-12-04T19:15:14,731 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-04T19:15:14,731 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-04T19:15:14,739 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/B/c7bd652b08ed43c0bcaef7a8dda50fcb 2024-12-04T19:15:14,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/C/3de4ad6ee4fb473ab6b2af742eeb2ac9 is 50, key is test_row_0/C:col10/1733339712543/Put/seqid=0 2024-12-04T19:15:14,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742136_1312 (size=12301) 2024-12-04T19:15:14,753 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:14,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37250 deadline: 1733339774750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:14,753 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:14,754 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:14,754 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:14,756 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:14,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37176 deadline: 1733339774753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:14,756 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:14,756 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:14,756 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:14,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37228 deadline: 1733339774753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:14,756 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:14,757 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:14,757 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:14,757 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:14,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-04T19:15:14,862 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:14,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37250 deadline: 1733339774862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:14,863 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:14,863 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:14,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37176 deadline: 1733339774862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:14,863 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:14,863 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:14,863 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:14,863 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:14,863 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:14,863 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:14,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37228 deadline: 1733339774862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:14,863 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:14,863 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:14,863 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:15,074 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:15,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37250 deadline: 1733339775071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:15,074 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:15,074 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:15,074 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:15,074 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:15,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37176 deadline: 1733339775072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:15,074 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:15,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37228 deadline: 1733339775072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:15,075 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:15,075 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:15,075 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:15,075 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:15,075 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:15,075 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:15,151 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/C/3de4ad6ee4fb473ab6b2af742eeb2ac9 2024-12-04T19:15:15,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/A/385292fa7a77471bb165da5bc1223c92 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/385292fa7a77471bb165da5bc1223c92 2024-12-04T19:15:15,157 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/385292fa7a77471bb165da5bc1223c92, entries=150, sequenceid=284, filesize=12.0 K 2024-12-04T19:15:15,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/B/c7bd652b08ed43c0bcaef7a8dda50fcb as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/c7bd652b08ed43c0bcaef7a8dda50fcb 2024-12-04T19:15:15,161 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/c7bd652b08ed43c0bcaef7a8dda50fcb, entries=150, sequenceid=284, filesize=12.0 K 2024-12-04T19:15:15,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/C/3de4ad6ee4fb473ab6b2af742eeb2ac9 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/3de4ad6ee4fb473ab6b2af742eeb2ac9 2024-12-04T19:15:15,164 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/3de4ad6ee4fb473ab6b2af742eeb2ac9, entries=150, sequenceid=284, filesize=12.0 K 2024-12-04T19:15:15,165 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 6c5c1603a070441ac38abd9294c86a68 in 864ms, sequenceid=284, compaction requested=true 2024-12-04T19:15:15,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for 6c5c1603a070441ac38abd9294c86a68: 2024-12-04T19:15:15,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. 2024-12-04T19:15:15,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-12-04T19:15:15,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-12-04T19:15:15,168 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-12-04T19:15:15,168 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0300 sec 2024-12-04T19:15:15,170 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 1.0350 sec 2024-12-04T19:15:15,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-04T19:15:15,272 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T19:15:15,272 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T19:15:15,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-12-04T19:15:15,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-04T19:15:15,274 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T19:15:15,274 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T19:15:15,274 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T19:15:15,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-04T19:15:15,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(8855): Flush requested on 6c5c1603a070441ac38abd9294c86a68 2024-12-04T19:15:15,384 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6c5c1603a070441ac38abd9294c86a68 3/3 column families, dataSize=140.89 KB heapSize=395.53 KB 2024-12-04T19:15:15,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c5c1603a070441ac38abd9294c86a68, store=A 2024-12-04T19:15:15,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T19:15:15,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c5c1603a070441ac38abd9294c86a68, store=B 2024-12-04T19:15:15,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T19:15:15,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c5c1603a070441ac38abd9294c86a68, store=C 2024-12-04T19:15:15,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T19:15:15,389 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/A/4d467314615c437cb477a768bf4f222e is 50, key is test_row_0/A:col10/1733339715382/Put/seqid=0 2024-12-04T19:15:15,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742137_1313 (size=14741) 2024-12-04T19:15:15,393 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/A/4d467314615c437cb477a768bf4f222e 2024-12-04T19:15:15,394 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:15,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:15,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37228 deadline: 1733339775391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:15,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37176 deadline: 1733339775390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:15,394 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:15,394 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:15,394 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:15,394 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:15,394 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:15,394 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:15,398 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:15,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37250 deadline: 1733339775395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:15,399 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:15,399 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:15,399 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:15,411 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/B/4b0cfdbd0f1e4e1eae3954e200218939 is 50, key is test_row_0/B:col10/1733339715382/Put/seqid=0 2024-12-04T19:15:15,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742138_1314 (size=12301) 2024-12-04T19:15:15,426 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-04T19:15:15,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. 2024-12-04T19:15:15,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. as already flushing 2024-12-04T19:15:15,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. 2024-12-04T19:15:15,427 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 6c5c1603a070441ac38abd9294c86a68, NAME => 'TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:15,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 6c5c1603a070441ac38abd9294c86a68, NAME => 'TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:15,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c5c1603a070441ac38abd9294c86a68, NAME => 'TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c5c1603a070441ac38abd9294c86a68, NAME => 'TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:15,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:15,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37228 deadline: 1733339775501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:15,504 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:15,504 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:15,504 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:15,504 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:15,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37176 deadline: 1733339775502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:15,504 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:15,504 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:15,504 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:15,504 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:15,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37250 deadline: 1733339775502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:15,505 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:15,505 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:15,505 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:15,579 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-04T19:15:15,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. 2024-12-04T19:15:15,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. as already flushing 2024-12-04T19:15:15,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. 2024-12-04T19:15:15,579 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 6c5c1603a070441ac38abd9294c86a68, NAME => 'TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:15,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 6c5c1603a070441ac38abd9294c86a68, NAME => 'TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:15,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c5c1603a070441ac38abd9294c86a68, NAME => 'TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c5c1603a070441ac38abd9294c86a68, NAME => 'TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:15,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-04T19:15:15,712 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:15,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37228 deadline: 1733339775711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:15,713 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:15,713 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:15,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37176 deadline: 1733339775712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:15,713 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:15,713 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:15,713 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:15,713 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:15,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37250 deadline: 1733339775712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:15,713 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:15,713 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:15,713 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:15,713 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:15,713 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:15,723 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:15,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37220 deadline: 1733339775722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:15,724 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 9ba2734cd39f,33969,1733339629791 for 'test_row_0' in 6c5c1603a070441ac38abd9294c86a68 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-04T19:15:15,724 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:15,724 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:15,724 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:15,731 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-04T19:15:15,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. 2024-12-04T19:15:15,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. as already flushing 2024-12-04T19:15:15,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. 2024-12-04T19:15:15,732 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 6c5c1603a070441ac38abd9294c86a68, NAME => 'TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:15,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 6c5c1603a070441ac38abd9294c86a68, NAME => 'TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:15,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c5c1603a070441ac38abd9294c86a68, NAME => 'TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c5c1603a070441ac38abd9294c86a68, NAME => 'TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:15,734 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:15,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37212 deadline: 1733339775731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:15,734 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 9ba2734cd39f,33969,1733339629791 for 'test_row_2' in 6c5c1603a070441ac38abd9294c86a68 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-04T19:15:15,734 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:15,734 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:15,735 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:15,817 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/B/4b0cfdbd0f1e4e1eae3954e200218939 2024-12-04T19:15:15,823 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/C/45b6b3baebed41c3bf81c48d273e471f is 50, key is test_row_0/C:col10/1733339715382/Put/seqid=0 2024-12-04T19:15:15,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742139_1315 (size=12301) 2024-12-04T19:15:15,833 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/C/45b6b3baebed41c3bf81c48d273e471f 2024-12-04T19:15:15,836 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/A/4d467314615c437cb477a768bf4f222e as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/4d467314615c437cb477a768bf4f222e 2024-12-04T19:15:15,840 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/4d467314615c437cb477a768bf4f222e, entries=200, sequenceid=320, filesize=14.4 K 2024-12-04T19:15:15,841 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/B/4b0cfdbd0f1e4e1eae3954e200218939 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/4b0cfdbd0f1e4e1eae3954e200218939 2024-12-04T19:15:15,844 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/4b0cfdbd0f1e4e1eae3954e200218939, entries=150, sequenceid=320, filesize=12.0 K 2024-12-04T19:15:15,845 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/C/45b6b3baebed41c3bf81c48d273e471f as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/45b6b3baebed41c3bf81c48d273e471f 2024-12-04T19:15:15,849 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/45b6b3baebed41c3bf81c48d273e471f, entries=150, sequenceid=320, filesize=12.0 K 2024-12-04T19:15:15,849 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~395.48 KB/404976, currentSize=40.25 KB/41220 for 6c5c1603a070441ac38abd9294c86a68 in 466ms, sequenceid=320, compaction requested=true 2024-12-04T19:15:15,849 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6c5c1603a070441ac38abd9294c86a68: 2024-12-04T19:15:15,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6c5c1603a070441ac38abd9294c86a68:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T19:15:15,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:15:15,849 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T19:15:15,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6c5c1603a070441ac38abd9294c86a68:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T19:15:15,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:15:15,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6c5c1603a070441ac38abd9294c86a68:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T19:15:15,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T19:15:15,850 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T19:15:15,851 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49600 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T19:15:15,851 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1541): 6c5c1603a070441ac38abd9294c86a68/B is initiating minor compaction (all files) 2024-12-04T19:15:15,851 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6c5c1603a070441ac38abd9294c86a68/B in TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. 2024-12-04T19:15:15,851 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/b5d0de0166b544f7ae1c0dc2b36a0ec1, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/66d74161b2c7444e8ac96c6224524518, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/c7bd652b08ed43c0bcaef7a8dda50fcb, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/4b0cfdbd0f1e4e1eae3954e200218939] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp, totalSize=48.4 K 2024-12-04T19:15:15,851 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52040 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T19:15:15,851 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting b5d0de0166b544f7ae1c0dc2b36a0ec1, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1733339711473 2024-12-04T19:15:15,851 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1541): 6c5c1603a070441ac38abd9294c86a68/A is initiating minor compaction (all files) 2024-12-04T19:15:15,851 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6c5c1603a070441ac38abd9294c86a68/A in TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. 2024-12-04T19:15:15,851 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/7f3c4e3a33ce41aeb2b009b13c7f75e0, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/872daeb46d904501be56a9dab8911cd8, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/385292fa7a77471bb165da5bc1223c92, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/4d467314615c437cb477a768bf4f222e] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp, totalSize=50.8 K 2024-12-04T19:15:15,852 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 66d74161b2c7444e8ac96c6224524518, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1733339711542 2024-12-04T19:15:15,852 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f3c4e3a33ce41aeb2b009b13c7f75e0, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1733339711473 2024-12-04T19:15:15,852 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 872daeb46d904501be56a9dab8911cd8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1733339711542 2024-12-04T19:15:15,852 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting c7bd652b08ed43c0bcaef7a8dda50fcb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1733339712543 2024-12-04T19:15:15,852 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 4b0cfdbd0f1e4e1eae3954e200218939, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1733339714739 2024-12-04T19:15:15,852 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 385292fa7a77471bb165da5bc1223c92, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1733339712543 2024-12-04T19:15:15,853 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d467314615c437cb477a768bf4f222e, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1733339714739 2024-12-04T19:15:15,862 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6c5c1603a070441ac38abd9294c86a68#A#compaction#270 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:15:15,862 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/A/b3b753a3b80b488b9b3d9b715c9fc77f is 50, key is test_row_0/A:col10/1733339715382/Put/seqid=0 2024-12-04T19:15:15,866 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6c5c1603a070441ac38abd9294c86a68#B#compaction#271 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:15:15,866 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/B/67f11390b1ee4ba6947f6d4322a09d14 is 50, key is test_row_0/B:col10/1733339715382/Put/seqid=0 2024-12-04T19:15:15,884 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-04T19:15:15,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. 2024-12-04T19:15:15,885 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing 6c5c1603a070441ac38abd9294c86a68 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-12-04T19:15:15,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c5c1603a070441ac38abd9294c86a68, store=A 2024-12-04T19:15:15,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:15:15,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c5c1603a070441ac38abd9294c86a68, store=B 2024-12-04T19:15:15,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:15:15,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c5c1603a070441ac38abd9294c86a68, store=C 2024-12-04T19:15:15,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:15:15,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742140_1316 (size=12983) 2024-12-04T19:15:15,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/A/61117ccc752e4b969e343b5a8b8df808 is 50, key is test_row_1/A:col10/1733339715393/Put/seqid=0 2024-12-04T19:15:15,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742141_1317 (size=12983) 2024-12-04T19:15:15,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-04T19:15:15,903 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/A/b3b753a3b80b488b9b3d9b715c9fc77f as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/b3b753a3b80b488b9b3d9b715c9fc77f 2024-12-04T19:15:15,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742142_1318 (size=9857) 2024-12-04T19:15:15,907 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6c5c1603a070441ac38abd9294c86a68/A of 6c5c1603a070441ac38abd9294c86a68 into b3b753a3b80b488b9b3d9b715c9fc77f(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:15:15,907 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6c5c1603a070441ac38abd9294c86a68: 2024-12-04T19:15:15,907 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., storeName=6c5c1603a070441ac38abd9294c86a68/A, priority=12, startTime=1733339715849; duration=0sec 2024-12-04T19:15:15,907 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T19:15:15,907 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6c5c1603a070441ac38abd9294c86a68:A 2024-12-04T19:15:15,907 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T19:15:15,908 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49600 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T19:15:15,908 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1541): 6c5c1603a070441ac38abd9294c86a68/C is initiating minor compaction (all files) 2024-12-04T19:15:15,908 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6c5c1603a070441ac38abd9294c86a68/C in TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. 2024-12-04T19:15:15,909 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/8a143da60c644a44b451bef3fa6ad266, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/b0964ab1e88a45cfa5079a7a2d6a683f, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/3de4ad6ee4fb473ab6b2af742eeb2ac9, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/45b6b3baebed41c3bf81c48d273e471f] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp, totalSize=48.4 K 2024-12-04T19:15:15,909 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a143da60c644a44b451bef3fa6ad266, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1733339711473 2024-12-04T19:15:15,909 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting b0964ab1e88a45cfa5079a7a2d6a683f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1733339711542 2024-12-04T19:15:15,909 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3de4ad6ee4fb473ab6b2af742eeb2ac9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1733339712543 2024-12-04T19:15:15,910 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 45b6b3baebed41c3bf81c48d273e471f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1733339714739 2024-12-04T19:15:15,920 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6c5c1603a070441ac38abd9294c86a68#C#compaction#273 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:15:15,920 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/C/8c9123c4e5174c4ea6fd93f36c99afe5 is 50, key is test_row_0/C:col10/1733339715382/Put/seqid=0 2024-12-04T19:15:15,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742143_1319 (size=12983) 2024-12-04T19:15:15,930 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/C/8c9123c4e5174c4ea6fd93f36c99afe5 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/8c9123c4e5174c4ea6fd93f36c99afe5 2024-12-04T19:15:15,941 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6c5c1603a070441ac38abd9294c86a68/C of 6c5c1603a070441ac38abd9294c86a68 into 8c9123c4e5174c4ea6fd93f36c99afe5(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:15:15,941 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6c5c1603a070441ac38abd9294c86a68: 2024-12-04T19:15:15,941 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., storeName=6c5c1603a070441ac38abd9294c86a68/C, priority=12, startTime=1733339715849; duration=0sec 2024-12-04T19:15:15,941 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:15:15,941 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6c5c1603a070441ac38abd9294c86a68:C 2024-12-04T19:15:16,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(8855): Flush requested on 6c5c1603a070441ac38abd9294c86a68 2024-12-04T19:15:16,024 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. as already flushing 2024-12-04T19:15:16,091 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733339716038, max timestamp=1733339716088Num uniques 859; 2024-12-04T19:15:16,091 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733339716038, max timestamp=1733339716088Num uniques 859; 2024-12-04T19:15:16,091 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T19:15:16,091 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T19:15:16,091 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733339716038, max timestamp=1733339716088Num uniques 859; 2024-12-04T19:15:16,091 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T19:15:16,126 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733339716117, max timestamp=1733339716123Num uniques 1068; 2024-12-04T19:15:16,126 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-12-04T19:15:16,127 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733339716117, max timestamp=1733339716123Num uniques 1068; 2024-12-04T19:15:16,127 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-12-04T19:15:16,127 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733339716117, max timestamp=1733339716123Num uniques 1068; 2024-12-04T19:15:16,127 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-12-04T19:15:16,159 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:16,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37176 deadline: 1733339776156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:16,160 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:16,160 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:16,160 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:16,161 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:16,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37228 deadline: 1733339776158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:16,161 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:16,162 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:16,162 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:16,163 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:16,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37250 deadline: 1733339776159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:16,163 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:16,163 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:16,163 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:16,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:16,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37176 deadline: 1733339776262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:16,265 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:16,265 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:16,265 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:16,276 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:16,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37228 deadline: 1733339776272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:16,276 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:16,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37250 deadline: 1733339776272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:16,276 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:16,276 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:16,276 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:16,276 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:16,277 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:16,277 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:16,303 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/B/67f11390b1ee4ba6947f6d4322a09d14 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/67f11390b1ee4ba6947f6d4322a09d14 2024-12-04T19:15:16,306 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/A/61117ccc752e4b969e343b5a8b8df808 2024-12-04T19:15:16,307 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6c5c1603a070441ac38abd9294c86a68/B of 6c5c1603a070441ac38abd9294c86a68 into 67f11390b1ee4ba6947f6d4322a09d14(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:15:16,307 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6c5c1603a070441ac38abd9294c86a68: 2024-12-04T19:15:16,307 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., storeName=6c5c1603a070441ac38abd9294c86a68/B, priority=12, startTime=1733339715849; duration=0sec 2024-12-04T19:15:16,307 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:15:16,307 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6c5c1603a070441ac38abd9294c86a68:B 2024-12-04T19:15:16,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/B/6592b52e005a460c8b880af047359665 is 50, key is test_row_1/B:col10/1733339715393/Put/seqid=0 2024-12-04T19:15:16,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742144_1320 (size=9857) 2024-12-04T19:15:16,318 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/B/6592b52e005a460c8b880af047359665 2024-12-04T19:15:16,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/C/170f889e88834407a206b138017d137f is 50, key is test_row_1/C:col10/1733339715393/Put/seqid=0 2024-12-04T19:15:16,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742145_1321 (size=9857) 2024-12-04T19:15:16,330 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/C/170f889e88834407a206b138017d137f 2024-12-04T19:15:16,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/A/61117ccc752e4b969e343b5a8b8df808 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/61117ccc752e4b969e343b5a8b8df808 2024-12-04T19:15:16,339 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/61117ccc752e4b969e343b5a8b8df808, entries=100, sequenceid=329, filesize=9.6 K 2024-12-04T19:15:16,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/B/6592b52e005a460c8b880af047359665 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/6592b52e005a460c8b880af047359665 2024-12-04T19:15:16,343 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/6592b52e005a460c8b880af047359665, entries=100, sequenceid=329, filesize=9.6 K 2024-12-04T19:15:16,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/C/170f889e88834407a206b138017d137f as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/170f889e88834407a206b138017d137f 2024-12-04T19:15:16,348 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/170f889e88834407a206b138017d137f, entries=100, sequenceid=329, filesize=9.6 K 2024-12-04T19:15:16,349 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for 6c5c1603a070441ac38abd9294c86a68 in 464ms, sequenceid=329, compaction requested=false 2024-12-04T19:15:16,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for 6c5c1603a070441ac38abd9294c86a68: 2024-12-04T19:15:16,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. 2024-12-04T19:15:16,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-12-04T19:15:16,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-12-04T19:15:16,352 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-12-04T19:15:16,352 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0760 sec 2024-12-04T19:15:16,354 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 1.0810 sec 2024-12-04T19:15:16,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-04T19:15:16,411 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T19:15:16,412 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T19:15:16,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-12-04T19:15:16,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-04T19:15:16,413 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T19:15:16,414 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T19:15:16,414 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T19:15:16,475 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6c5c1603a070441ac38abd9294c86a68 3/3 column families, dataSize=147.60 KB heapSize=420 KB 2024-12-04T19:15:16,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c5c1603a070441ac38abd9294c86a68, store=A 2024-12-04T19:15:16,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-04T19:15:16,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c5c1603a070441ac38abd9294c86a68, store=B 2024-12-04T19:15:16,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(8855): Flush requested on 6c5c1603a070441ac38abd9294c86a68 2024-12-04T19:15:16,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-04T19:15:16,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c5c1603a070441ac38abd9294c86a68, store=C 2024-12-04T19:15:16,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-04T19:15:16,480 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/A/67ba12cacd1a48cbacc212b6f6d0e8c8 is 50, key is test_row_0/A:col10/1733339716156/Put/seqid=0 2024-12-04T19:15:16,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742146_1322 (size=14741) 2024-12-04T19:15:16,485 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=387 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/A/67ba12cacd1a48cbacc212b6f6d0e8c8 2024-12-04T19:15:16,489 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:16,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37228 deadline: 1733339776485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:16,489 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:16,489 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:16,489 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:16,489 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:16,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37250 deadline: 1733339776486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:16,490 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:16,490 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:16,490 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:16,491 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:16,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37176 deadline: 1733339776487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:16,491 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:16,491 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:16,491 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:16,492 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/B/61ff286b02c34d24ae2793dd25092397 is 50, key is test_row_0/B:col10/1733339716156/Put/seqid=0 2024-12-04T19:15:16,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742147_1323 (size=12301) 2024-12-04T19:15:16,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-04T19:15:16,566 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-12-04T19:15:16,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. 2024-12-04T19:15:16,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. as already flushing 2024-12-04T19:15:16,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. 2024-12-04T19:15:16,566 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 6c5c1603a070441ac38abd9294c86a68, NAME => 'TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:16,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 6c5c1603a070441ac38abd9294c86a68, NAME => 'TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:16,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c5c1603a070441ac38abd9294c86a68, NAME => 'TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c5c1603a070441ac38abd9294c86a68, NAME => 'TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:16,593 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:16,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37228 deadline: 1733339776592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:16,594 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:16,594 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:16,594 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:16,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37250 deadline: 1733339776592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:16,594 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:16,594 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:16,594 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:16,594 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:16,604 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:16,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37176 deadline: 1733339776602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:16,604 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:16,605 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:16,605 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:16,718 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-12-04T19:15:16,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. 2024-12-04T19:15:16,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. as already flushing 2024-12-04T19:15:16,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. 2024-12-04T19:15:16,719 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 6c5c1603a070441ac38abd9294c86a68, NAME => 'TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:16,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 6c5c1603a070441ac38abd9294c86a68, NAME => 'TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:16,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c5c1603a070441ac38abd9294c86a68, NAME => 'TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c5c1603a070441ac38abd9294c86a68, NAME => 'TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:16,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-04T19:15:16,804 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:16,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37228 deadline: 1733339776802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:16,804 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:16,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37250 deadline: 1733339776802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:16,804 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:16,804 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:16,804 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:16,804 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:16,804 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:16,804 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:16,814 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:16,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37176 deadline: 1733339776812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:16,814 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:16,814 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:16,814 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:16,872 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-12-04T19:15:16,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. 2024-12-04T19:15:16,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. as already flushing 2024-12-04T19:15:16,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. 2024-12-04T19:15:16,872 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 6c5c1603a070441ac38abd9294c86a68, NAME => 'TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:16,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 6c5c1603a070441ac38abd9294c86a68, NAME => 'TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:16,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c5c1603a070441ac38abd9294c86a68, NAME => 'TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c5c1603a070441ac38abd9294c86a68, NAME => 'TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:16,896 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=387 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/B/61ff286b02c34d24ae2793dd25092397 2024-12-04T19:15:16,901 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/C/196456b5a3de44b5b64e72365f966577 is 50, key is test_row_0/C:col10/1733339716156/Put/seqid=0 2024-12-04T19:15:16,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742148_1324 (size=12301) 2024-12-04T19:15:17,024 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-12-04T19:15:17,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. 2024-12-04T19:15:17,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. as already flushing 2024-12-04T19:15:17,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. 2024-12-04T19:15:17,025 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 6c5c1603a070441ac38abd9294c86a68, NAME => 'TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:17,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 6c5c1603a070441ac38abd9294c86a68, NAME => 'TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:17,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c5c1603a070441ac38abd9294c86a68, NAME => 'TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c5c1603a070441ac38abd9294c86a68, NAME => 'TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:17,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-04T19:15:17,112 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:17,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37228 deadline: 1733339777111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:17,112 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:17,113 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:17,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37250 deadline: 1733339777111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:17,113 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:17,113 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:17,113 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:17,113 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:17,113 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:17,122 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:17,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37176 deadline: 1733339777122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:17,122 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:17,122 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:17,122 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:17,177 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-12-04T19:15:17,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. 2024-12-04T19:15:17,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. as already flushing 2024-12-04T19:15:17,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. 2024-12-04T19:15:17,177 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 6c5c1603a070441ac38abd9294c86a68, NAME => 'TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:17,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 6c5c1603a070441ac38abd9294c86a68, NAME => 'TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:17,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c5c1603a070441ac38abd9294c86a68, NAME => 'TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c5c1603a070441ac38abd9294c86a68, NAME => 'TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:17,305 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=387 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/C/196456b5a3de44b5b64e72365f966577 2024-12-04T19:15:17,310 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/A/67ba12cacd1a48cbacc212b6f6d0e8c8 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/67ba12cacd1a48cbacc212b6f6d0e8c8 2024-12-04T19:15:17,313 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/67ba12cacd1a48cbacc212b6f6d0e8c8, entries=200, sequenceid=387, filesize=14.4 K 2024-12-04T19:15:17,313 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/B/61ff286b02c34d24ae2793dd25092397 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/61ff286b02c34d24ae2793dd25092397 2024-12-04T19:15:17,317 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/61ff286b02c34d24ae2793dd25092397, entries=150, sequenceid=387, filesize=12.0 K 2024-12-04T19:15:17,317 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/C/196456b5a3de44b5b64e72365f966577 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/196456b5a3de44b5b64e72365f966577 2024-12-04T19:15:17,320 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/196456b5a3de44b5b64e72365f966577, entries=150, sequenceid=387, filesize=12.0 K 2024-12-04T19:15:17,321 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~419.95 KB/430032, currentSize=33.54 KB/34350 for 6c5c1603a070441ac38abd9294c86a68 in 846ms, sequenceid=387, compaction requested=true 2024-12-04T19:15:17,321 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6c5c1603a070441ac38abd9294c86a68: 2024-12-04T19:15:17,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6c5c1603a070441ac38abd9294c86a68:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T19:15:17,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:15:17,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6c5c1603a070441ac38abd9294c86a68:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T19:15:17,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:15:17,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6c5c1603a070441ac38abd9294c86a68:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T19:15:17,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T19:15:17,322 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T19:15:17,322 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T19:15:17,322 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T19:15:17,322 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37581 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T19:15:17,322 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1541): 6c5c1603a070441ac38abd9294c86a68/A is initiating minor compaction (all files) 2024-12-04T19:15:17,322 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1541): 6c5c1603a070441ac38abd9294c86a68/B is initiating minor compaction (all files) 2024-12-04T19:15:17,322 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6c5c1603a070441ac38abd9294c86a68/A in TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. 2024-12-04T19:15:17,322 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6c5c1603a070441ac38abd9294c86a68/B in TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. 2024-12-04T19:15:17,323 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/67f11390b1ee4ba6947f6d4322a09d14, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/6592b52e005a460c8b880af047359665, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/61ff286b02c34d24ae2793dd25092397] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp, totalSize=34.3 K 2024-12-04T19:15:17,323 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/b3b753a3b80b488b9b3d9b715c9fc77f, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/61117ccc752e4b969e343b5a8b8df808, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/67ba12cacd1a48cbacc212b6f6d0e8c8] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp, totalSize=36.7 K 2024-12-04T19:15:17,323 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting b3b753a3b80b488b9b3d9b715c9fc77f, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1733339714739 2024-12-04T19:15:17,323 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 67f11390b1ee4ba6947f6d4322a09d14, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1733339714739 2024-12-04T19:15:17,323 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 61117ccc752e4b969e343b5a8b8df808, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1733339715384 2024-12-04T19:15:17,323 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 6592b52e005a460c8b880af047359665, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1733339715384 2024-12-04T19:15:17,323 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 61ff286b02c34d24ae2793dd25092397, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1733339716154 2024-12-04T19:15:17,323 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 67ba12cacd1a48cbacc212b6f6d0e8c8, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1733339716154 2024-12-04T19:15:17,329 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6c5c1603a070441ac38abd9294c86a68#A#compaction#279 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:15:17,329 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6c5c1603a070441ac38abd9294c86a68#B#compaction#280 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:15:17,329 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/A/ee93cd6bc3af4d7ca4522ff6a9335299 is 50, key is test_row_0/A:col10/1733339716156/Put/seqid=0 2024-12-04T19:15:17,329 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/B/6a8d0d242a5e46f48c01e09e85b3a48b is 50, key is test_row_0/B:col10/1733339716156/Put/seqid=0 2024-12-04T19:15:17,330 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-12-04T19:15:17,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. 2024-12-04T19:15:17,330 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing 6c5c1603a070441ac38abd9294c86a68 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-04T19:15:17,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c5c1603a070441ac38abd9294c86a68, store=A 2024-12-04T19:15:17,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:15:17,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c5c1603a070441ac38abd9294c86a68, store=B 2024-12-04T19:15:17,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:15:17,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c5c1603a070441ac38abd9294c86a68, store=C 2024-12-04T19:15:17,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:15:17,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742150_1326 (size=13085) 2024-12-04T19:15:17,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742149_1325 (size=13085) 2024-12-04T19:15:17,354 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/B/6a8d0d242a5e46f48c01e09e85b3a48b as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/6a8d0d242a5e46f48c01e09e85b3a48b 2024-12-04T19:15:17,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/A/afee1509bf6a4606a100a5cd40a24a2a is 50, key is test_row_0/A:col10/1733339716486/Put/seqid=0 2024-12-04T19:15:17,359 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6c5c1603a070441ac38abd9294c86a68/B of 6c5c1603a070441ac38abd9294c86a68 into 6a8d0d242a5e46f48c01e09e85b3a48b(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:15:17,359 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6c5c1603a070441ac38abd9294c86a68: 2024-12-04T19:15:17,359 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., storeName=6c5c1603a070441ac38abd9294c86a68/B, priority=13, startTime=1733339717321; duration=0sec 2024-12-04T19:15:17,359 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T19:15:17,359 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6c5c1603a070441ac38abd9294c86a68:B 2024-12-04T19:15:17,360 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T19:15:17,361 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T19:15:17,361 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1541): 6c5c1603a070441ac38abd9294c86a68/C is initiating minor compaction (all files) 2024-12-04T19:15:17,361 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6c5c1603a070441ac38abd9294c86a68/C in TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. 2024-12-04T19:15:17,361 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/8c9123c4e5174c4ea6fd93f36c99afe5, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/170f889e88834407a206b138017d137f, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/196456b5a3de44b5b64e72365f966577] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp, totalSize=34.3 K 2024-12-04T19:15:17,361 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c9123c4e5174c4ea6fd93f36c99afe5, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1733339714739 2024-12-04T19:15:17,361 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 170f889e88834407a206b138017d137f, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1733339715384 2024-12-04T19:15:17,362 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 196456b5a3de44b5b64e72365f966577, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1733339716154 2024-12-04T19:15:17,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742151_1327 (size=12301) 2024-12-04T19:15:17,374 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6c5c1603a070441ac38abd9294c86a68#C#compaction#282 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:15:17,374 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/C/80eb6b87beec4bc28682060f42bde326 is 50, key is test_row_0/C:col10/1733339716156/Put/seqid=0 2024-12-04T19:15:17,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742152_1328 (size=13085) 2024-12-04T19:15:17,381 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/C/80eb6b87beec4bc28682060f42bde326 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/80eb6b87beec4bc28682060f42bde326 2024-12-04T19:15:17,385 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6c5c1603a070441ac38abd9294c86a68/C of 6c5c1603a070441ac38abd9294c86a68 into 80eb6b87beec4bc28682060f42bde326(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:15:17,385 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6c5c1603a070441ac38abd9294c86a68: 2024-12-04T19:15:17,385 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., storeName=6c5c1603a070441ac38abd9294c86a68/C, priority=13, startTime=1733339717321; duration=0sec 2024-12-04T19:15:17,385 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:15:17,385 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6c5c1603a070441ac38abd9294c86a68:C 2024-12-04T19:15:17,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-04T19:15:17,623 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. as already flushing 2024-12-04T19:15:17,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(8855): Flush requested on 6c5c1603a070441ac38abd9294c86a68 2024-12-04T19:15:17,649 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733339717641, max timestamp=1733339717646Num uniques 859; 2024-12-04T19:15:17,649 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733339717641, max timestamp=1733339717646Num uniques 859; 2024-12-04T19:15:17,649 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T19:15:17,649 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T19:15:17,649 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733339717641, max timestamp=1733339717646Num uniques 859; 2024-12-04T19:15:17,649 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T19:15:17,672 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733339717661, max timestamp=1733339717669Num uniques 1018; 2024-12-04T19:15:17,672 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-04T19:15:17,672 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733339717661, max timestamp=1733339717669Num uniques 1018; 2024-12-04T19:15:17,672 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-04T19:15:17,672 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733339717661, max timestamp=1733339717669Num uniques 1018; 2024-12-04T19:15:17,672 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-04T19:15:17,696 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:17,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37176 deadline: 1733339777693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:17,696 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:17,696 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:17,697 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:17,697 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1733339717683, max timestamp=1733339717694Num uniques 1077; 2024-12-04T19:15:17,697 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1733339717683, max timestamp=1733339717694Num uniques 1077; 2024-12-04T19:15:17,697 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1733339717683, max timestamp=1733339717694Num uniques 1077; 2024-12-04T19:15:17,697 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=12682, suffix heap size=139976,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=277 2024-12-04T19:15:17,697 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=12682, suffix heap size=139976,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=277 2024-12-04T19:15:17,697 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=12682, suffix heap size=139976,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=277 2024-12-04T19:15:17,727 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:17,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37250 deadline: 1733339777724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:17,728 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:17,728 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:17,728 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:17,729 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:17,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37228 deadline: 1733339777727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:17,730 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:17,730 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:17,730 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:17,746 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/A/ee93cd6bc3af4d7ca4522ff6a9335299 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/ee93cd6bc3af4d7ca4522ff6a9335299 2024-12-04T19:15:17,750 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6c5c1603a070441ac38abd9294c86a68/A of 6c5c1603a070441ac38abd9294c86a68 into ee93cd6bc3af4d7ca4522ff6a9335299(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:15:17,750 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6c5c1603a070441ac38abd9294c86a68: 2024-12-04T19:15:17,750 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., storeName=6c5c1603a070441ac38abd9294c86a68/A, priority=13, startTime=1733339717321; duration=0sec 2024-12-04T19:15:17,750 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:15:17,750 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6c5c1603a070441ac38abd9294c86a68:A 2024-12-04T19:15:17,765 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/A/afee1509bf6a4606a100a5cd40a24a2a 2024-12-04T19:15:17,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/B/e43041abf46349a49601cfcc3f0e78f1 is 50, key is test_row_0/B:col10/1733339716486/Put/seqid=0 2024-12-04T19:15:17,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742153_1329 (size=12301) 2024-12-04T19:15:17,774 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/B/e43041abf46349a49601cfcc3f0e78f1 2024-12-04T19:15:17,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/C/acbb3fb7b7f8419483297cda88fd0470 is 50, key is test_row_0/C:col10/1733339716486/Put/seqid=0 2024-12-04T19:15:17,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742154_1330 (size=12301) 2024-12-04T19:15:17,805 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:17,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37176 deadline: 1733339777801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:17,806 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:17,806 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:17,806 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:17,835 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:17,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37250 deadline: 1733339777832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:17,835 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:17,835 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:17,835 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:17,843 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:17,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37228 deadline: 1733339777842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:17,844 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:17,844 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:17,844 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:18,012 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:18,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37176 deadline: 1733339778011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:18,013 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:18,013 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:18,013 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:18,014 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-04T19:15:18,043 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:18,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37250 deadline: 1733339778042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:18,043 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:18,043 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:18,043 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:18,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:18,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37228 deadline: 1733339778052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:18,055 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:18,055 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:18,055 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:18,183 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/C/acbb3fb7b7f8419483297cda88fd0470 2024-12-04T19:15:18,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/A/afee1509bf6a4606a100a5cd40a24a2a as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/afee1509bf6a4606a100a5cd40a24a2a 2024-12-04T19:15:18,189 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/afee1509bf6a4606a100a5cd40a24a2a, entries=150, sequenceid=395, filesize=12.0 K 2024-12-04T19:15:18,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/B/e43041abf46349a49601cfcc3f0e78f1 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/e43041abf46349a49601cfcc3f0e78f1 2024-12-04T19:15:18,193 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/e43041abf46349a49601cfcc3f0e78f1, entries=150, sequenceid=395, filesize=12.0 K 2024-12-04T19:15:18,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/C/acbb3fb7b7f8419483297cda88fd0470 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/acbb3fb7b7f8419483297cda88fd0470 2024-12-04T19:15:18,196 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/acbb3fb7b7f8419483297cda88fd0470, entries=150, sequenceid=395, filesize=12.0 K 2024-12-04T19:15:18,197 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=147.60 KB/151140 for 6c5c1603a070441ac38abd9294c86a68 in 867ms, sequenceid=395, compaction requested=false 2024-12-04T19:15:18,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for 6c5c1603a070441ac38abd9294c86a68: 2024-12-04T19:15:18,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. 2024-12-04T19:15:18,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-12-04T19:15:18,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-12-04T19:15:18,200 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-12-04T19:15:18,200 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7840 sec 2024-12-04T19:15:18,202 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 1.7890 sec 2024-12-04T19:15:18,203 INFO [Thread-1190 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1190. 2024-12-04T19:15:18,203 DEBUG [Thread-1190 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T19:15:18,203 DEBUG [Thread-1190 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:15:18,203 DEBUG [Thread-1190 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:15:18,203 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T19:15:18,204 INFO [Thread-1192 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1192. 2024-12-04T19:15:18,204 DEBUG [Thread-1192 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T19:15:18,204 DEBUG [Thread-1192 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:15:18,204 DEBUG [Thread-1192 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:15:18,204 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T19:15:18,205 INFO [Thread-1196 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1196. 2024-12-04T19:15:18,205 INFO [Thread-1198 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1198. 2024-12-04T19:15:18,205 DEBUG [Thread-1198 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T19:15:18,205 DEBUG [Thread-1196 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T19:15:18,205 DEBUG [Thread-1198 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:15:18,205 DEBUG [Thread-1196 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:15:18,206 INFO [Thread-1194 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1194. 2024-12-04T19:15:18,206 DEBUG [Thread-1198 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:15:18,206 DEBUG [Thread-1196 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:15:18,206 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T19:15:18,206 DEBUG [Thread-1194 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T19:15:18,206 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T19:15:18,206 DEBUG [Thread-1194 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:15:18,206 DEBUG [Thread-1194 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:15:18,206 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T19:15:18,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(8855): Flush requested on 6c5c1603a070441ac38abd9294c86a68 2024-12-04T19:15:18,325 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6c5c1603a070441ac38abd9294c86a68 3/3 column families, dataSize=154.31 KB heapSize=434.39 KB 2024-12-04T19:15:18,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c5c1603a070441ac38abd9294c86a68, store=A 2024-12-04T19:15:18,325 INFO [Thread-1185 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1185. 2024-12-04T19:15:18,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-04T19:15:18,325 DEBUG [Thread-1185 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T19:15:18,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c5c1603a070441ac38abd9294c86a68, store=B 2024-12-04T19:15:18,325 DEBUG [Thread-1185 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:15:18,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-04T19:15:18,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c5c1603a070441ac38abd9294c86a68, store=C 2024-12-04T19:15:18,326 DEBUG [Thread-1185 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:15:18,326 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-04T19:15:18,326 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T19:15:18,330 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/A/566ca0b2462145d0a41d21e701ca4dea is 50, key is test_row_0/A:col10/1733339718323/Put/seqid=0 2024-12-04T19:15:18,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742155_1331 (size=12301) 2024-12-04T19:15:18,353 INFO [Thread-1183 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1183. 2024-12-04T19:15:18,354 DEBUG [Thread-1183 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T19:15:18,354 DEBUG [Thread-1183 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:15:18,354 DEBUG [Thread-1183 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:15:18,354 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T19:15:18,364 INFO [Thread-1187 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1187. 2024-12-04T19:15:18,364 DEBUG [Thread-1187 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T19:15:18,364 DEBUG [Thread-1187 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:15:18,365 DEBUG [Thread-1187 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:15:18,365 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T19:15:18,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-04T19:15:18,573 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T19:15:18,736 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=470 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/A/566ca0b2462145d0a41d21e701ca4dea 2024-12-04T19:15:18,809 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/B/6acc9d08422b423ebac03db8b7e6b32a is 50, key is test_row_0/B:col10/1733339718323/Put/seqid=0 2024-12-04T19:15:18,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742156_1332 (size=12301) 2024-12-04T19:15:19,215 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=470 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/B/6acc9d08422b423ebac03db8b7e6b32a 2024-12-04T19:15:19,227 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/C/f57f3e15f4b244328fa6f982eef3321c is 50, key is test_row_0/C:col10/1733339718323/Put/seqid=0 2024-12-04T19:15:19,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742157_1333 (size=12301) 2024-12-04T19:15:19,632 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=470 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/C/f57f3e15f4b244328fa6f982eef3321c 2024-12-04T19:15:19,636 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/A/566ca0b2462145d0a41d21e701ca4dea as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/566ca0b2462145d0a41d21e701ca4dea 2024-12-04T19:15:19,639 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/566ca0b2462145d0a41d21e701ca4dea, entries=150, sequenceid=470, filesize=12.0 K 2024-12-04T19:15:19,640 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/B/6acc9d08422b423ebac03db8b7e6b32a as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/6acc9d08422b423ebac03db8b7e6b32a 2024-12-04T19:15:19,643 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/6acc9d08422b423ebac03db8b7e6b32a, entries=150, sequenceid=470, filesize=12.0 K 2024-12-04T19:15:19,644 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/C/f57f3e15f4b244328fa6f982eef3321c as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/f57f3e15f4b244328fa6f982eef3321c 2024-12-04T19:15:19,647 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/f57f3e15f4b244328fa6f982eef3321c, entries=150, sequenceid=470, filesize=12.0 K 2024-12-04T19:15:19,648 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~434.34 KB/444768, currentSize=13.42 KB/13740 for 6c5c1603a070441ac38abd9294c86a68 in 1322ms, sequenceid=470, compaction requested=true 2024-12-04T19:15:19,648 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6c5c1603a070441ac38abd9294c86a68: 2024-12-04T19:15:19,648 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6c5c1603a070441ac38abd9294c86a68:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T19:15:19,648 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:15:19,648 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6c5c1603a070441ac38abd9294c86a68:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T19:15:19,648 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:15:19,648 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T19:15:19,648 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6c5c1603a070441ac38abd9294c86a68:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T19:15:19,648 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T19:15:19,648 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T19:15:19,649 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T19:15:19,649 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T19:15:19,649 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1541): 6c5c1603a070441ac38abd9294c86a68/A is initiating minor compaction (all files) 2024-12-04T19:15:19,649 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1541): 6c5c1603a070441ac38abd9294c86a68/B is initiating minor compaction (all files) 2024-12-04T19:15:19,649 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6c5c1603a070441ac38abd9294c86a68/A in TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. 2024-12-04T19:15:19,649 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6c5c1603a070441ac38abd9294c86a68/B in TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. 2024-12-04T19:15:19,649 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/ee93cd6bc3af4d7ca4522ff6a9335299, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/afee1509bf6a4606a100a5cd40a24a2a, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/566ca0b2462145d0a41d21e701ca4dea] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp, totalSize=36.8 K 2024-12-04T19:15:19,649 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/6a8d0d242a5e46f48c01e09e85b3a48b, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/e43041abf46349a49601cfcc3f0e78f1, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/6acc9d08422b423ebac03db8b7e6b32a] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp, totalSize=36.8 K 2024-12-04T19:15:19,649 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a8d0d242a5e46f48c01e09e85b3a48b, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1733339716154 2024-12-04T19:15:19,649 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting ee93cd6bc3af4d7ca4522ff6a9335299, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1733339716154 2024-12-04T19:15:19,649 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting afee1509bf6a4606a100a5cd40a24a2a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1733339716483 2024-12-04T19:15:19,649 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting e43041abf46349a49601cfcc3f0e78f1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1733339716483 2024-12-04T19:15:19,650 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 566ca0b2462145d0a41d21e701ca4dea, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=470, earliestPutTs=1733339717720 2024-12-04T19:15:19,650 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 6acc9d08422b423ebac03db8b7e6b32a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=470, earliestPutTs=1733339717720 2024-12-04T19:15:19,655 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6c5c1603a070441ac38abd9294c86a68#A#compaction#288 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:15:19,655 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6c5c1603a070441ac38abd9294c86a68#B#compaction#289 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:15:19,656 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/B/386e5842914d4c8abcd910bd0d0644e6 is 50, key is test_row_0/B:col10/1733339718323/Put/seqid=0 2024-12-04T19:15:19,656 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/A/ba1e994b9121457297b8cb48a85bf750 is 50, key is test_row_0/A:col10/1733339718323/Put/seqid=0 2024-12-04T19:15:19,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742159_1335 (size=13187) 2024-12-04T19:15:19,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742158_1334 (size=13187) 2024-12-04T19:15:19,744 INFO [Thread-1181 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1181. 2024-12-04T19:15:19,744 DEBUG [Thread-1181 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T19:15:19,744 DEBUG [Thread-1181 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:15:19,744 DEBUG [Thread-1181 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:15:19,744 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T19:15:19,753 INFO [Thread-1179 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1179. 2024-12-04T19:15:19,753 DEBUG [Thread-1179 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T19:15:19,753 DEBUG [Thread-1179 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:15:19,753 DEBUG [Thread-1179 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:15:19,753 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T19:15:19,753 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-04T19:15:19,753 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 53 2024-12-04T19:15:19,753 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 58 2024-12-04T19:15:19,753 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 96 2024-12-04T19:15:19,753 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 87 2024-12-04T19:15:19,753 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 101 2024-12-04T19:15:19,753 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-04T19:15:19,753 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4419 2024-12-04T19:15:19,753 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4394 2024-12-04T19:15:19,753 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4292 2024-12-04T19:15:19,753 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4400 2024-12-04T19:15:19,753 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4405 2024-12-04T19:15:19,753 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-04T19:15:19,753 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-04T19:15:19,753 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T19:15:19,753 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:15:19,753 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:15:19,753 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T19:15:19,754 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-04T19:15:19,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-04T19:15:19,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-04T19:15:19,756 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733339719756"}]},"ts":"1733339719756"} 2024-12-04T19:15:19,757 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-04T19:15:19,757 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-04T19:15:19,758 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-04T19:15:19,759 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6c5c1603a070441ac38abd9294c86a68, UNASSIGN}] 2024-12-04T19:15:19,759 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6c5c1603a070441ac38abd9294c86a68, UNASSIGN 2024-12-04T19:15:19,760 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=6c5c1603a070441ac38abd9294c86a68, regionState=CLOSING, regionLocation=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:19,761 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6c5c1603a070441ac38abd9294c86a68, UNASSIGN because future has completed 2024-12-04T19:15:19,761 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-04T19:15:19,761 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791}] 2024-12-04T19:15:19,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-04T19:15:19,914 INFO [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(122): Close 6c5c1603a070441ac38abd9294c86a68 2024-12-04T19:15:19,914 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-04T19:15:19,915 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1722): Closing 6c5c1603a070441ac38abd9294c86a68, disabling compactions & flushes 2024-12-04T19:15:19,915 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1993): waiting for 2 compactions to complete for region TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. 2024-12-04T19:15:20,071 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/B/386e5842914d4c8abcd910bd0d0644e6 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/386e5842914d4c8abcd910bd0d0644e6 2024-12-04T19:15:20,071 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/A/ba1e994b9121457297b8cb48a85bf750 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/ba1e994b9121457297b8cb48a85bf750 2024-12-04T19:15:20,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-04T19:15:20,077 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6c5c1603a070441ac38abd9294c86a68/A of 6c5c1603a070441ac38abd9294c86a68 into ba1e994b9121457297b8cb48a85bf750(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:15:20,077 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6c5c1603a070441ac38abd9294c86a68/B of 6c5c1603a070441ac38abd9294c86a68 into 386e5842914d4c8abcd910bd0d0644e6(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:15:20,077 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6c5c1603a070441ac38abd9294c86a68: 2024-12-04T19:15:20,077 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6c5c1603a070441ac38abd9294c86a68: 2024-12-04T19:15:20,077 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., storeName=6c5c1603a070441ac38abd9294c86a68/A, priority=13, startTime=1733339719648; duration=0sec 2024-12-04T19:15:20,077 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., storeName=6c5c1603a070441ac38abd9294c86a68/B, priority=13, startTime=1733339719648; duration=0sec 2024-12-04T19:15:20,077 INFO [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. 2024-12-04T19:15:20,077 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. 2024-12-04T19:15:20,077 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T19:15:20,077 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T19:15:20,077 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6c5c1603a070441ac38abd9294c86a68:A 2024-12-04T19:15:20,077 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6c5c1603a070441ac38abd9294c86a68:B 2024-12-04T19:15:20,077 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. after waiting 0 ms 2024-12-04T19:15:20,077 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. 2024-12-04T19:15:20,077 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. because compaction request was cancelled 2024-12-04T19:15:20,077 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6c5c1603a070441ac38abd9294c86a68:C 2024-12-04T19:15:20,077 INFO [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(2902): Flushing 6c5c1603a070441ac38abd9294c86a68 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-12-04T19:15:20,077 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c5c1603a070441ac38abd9294c86a68, store=A 2024-12-04T19:15:20,077 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:15:20,077 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c5c1603a070441ac38abd9294c86a68, store=B 2024-12-04T19:15:20,077 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:15:20,077 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c5c1603a070441ac38abd9294c86a68, store=C 2024-12-04T19:15:20,078 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:15:20,081 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/A/11015cd8073944008663bb4cb2378022 is 50, key is test_row_0/A:col10/1733339719742/Put/seqid=0 2024-12-04T19:15:20,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742160_1336 (size=9857) 2024-12-04T19:15:20,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-04T19:15:20,487 INFO [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=479 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/A/11015cd8073944008663bb4cb2378022 2024-12-04T19:15:20,501 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/B/674dcf059dca4b9f8176c0fec525654a is 50, key is test_row_0/B:col10/1733339719742/Put/seqid=0 2024-12-04T19:15:20,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742161_1337 (size=9857) 2024-12-04T19:15:20,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-04T19:15:20,906 INFO [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=479 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/B/674dcf059dca4b9f8176c0fec525654a 2024-12-04T19:15:20,918 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/C/22153514dffd4ab5ab4f1184cc89b39d is 50, key is test_row_0/C:col10/1733339719742/Put/seqid=0 2024-12-04T19:15:20,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742162_1338 (size=9857) 2024-12-04T19:15:21,323 INFO [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=479 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/C/22153514dffd4ab5ab4f1184cc89b39d 2024-12-04T19:15:21,330 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/A/11015cd8073944008663bb4cb2378022 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/11015cd8073944008663bb4cb2378022 2024-12-04T19:15:21,336 INFO [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/11015cd8073944008663bb4cb2378022, entries=100, sequenceid=479, filesize=9.6 K 2024-12-04T19:15:21,337 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/B/674dcf059dca4b9f8176c0fec525654a as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/674dcf059dca4b9f8176c0fec525654a 2024-12-04T19:15:21,341 INFO [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/674dcf059dca4b9f8176c0fec525654a, entries=100, sequenceid=479, filesize=9.6 K 2024-12-04T19:15:21,342 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/.tmp/C/22153514dffd4ab5ab4f1184cc89b39d as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/22153514dffd4ab5ab4f1184cc89b39d 2024-12-04T19:15:21,345 INFO [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/22153514dffd4ab5ab4f1184cc89b39d, entries=100, sequenceid=479, filesize=9.6 K 2024-12-04T19:15:21,346 INFO [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 6c5c1603a070441ac38abd9294c86a68 in 1268ms, sequenceid=479, compaction requested=true 2024-12-04T19:15:21,346 DEBUG [StoreCloser-TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/a3d57a61ea6e43dab1cb710687593cd0, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/2465597156e04dd7a648a214d70358a9, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/5ccd8ab462434030b34f7d2b2d48df18, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/41c356f67a9543d5ae9128d5d78580dd, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/d45b55e78f644e5dabaf814d41f4e035, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/27d763afc46340a5a5d4f541d357e7a0, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/4ecc07e726d045dd8e95c474560b5d54, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/21ea5931056f45d08b8348e5c21f67dd, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/95e105114fcd4d758e6c13c6c2de5480, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/b90855e589394347808aea074cc66c2f, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/17a6d85075974b8e8a5b51fe16ab9c40, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/14fa4989bbdb4ccc9b067726dac45d6f, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/c1a884816070474cbd3a878ff3a6a32e, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/fb281ae36a1949a1952ac2b5f45f7ea7, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/4488084553434cedb90010ec357e5a13, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/7f3c4e3a33ce41aeb2b009b13c7f75e0, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/a0d776413f1d4bc38705cf93911376aa, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/872daeb46d904501be56a9dab8911cd8, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/385292fa7a77471bb165da5bc1223c92, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/4d467314615c437cb477a768bf4f222e, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/b3b753a3b80b488b9b3d9b715c9fc77f, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/61117ccc752e4b969e343b5a8b8df808, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/67ba12cacd1a48cbacc212b6f6d0e8c8, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/ee93cd6bc3af4d7ca4522ff6a9335299, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/afee1509bf6a4606a100a5cd40a24a2a, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/566ca0b2462145d0a41d21e701ca4dea] to archive 2024-12-04T19:15:21,347 DEBUG [StoreCloser-TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-04T19:15:21,349 DEBUG [StoreCloser-TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/a3d57a61ea6e43dab1cb710687593cd0 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/a3d57a61ea6e43dab1cb710687593cd0 2024-12-04T19:15:21,350 DEBUG [StoreCloser-TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/2465597156e04dd7a648a214d70358a9 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/2465597156e04dd7a648a214d70358a9 2024-12-04T19:15:21,351 DEBUG [StoreCloser-TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/5ccd8ab462434030b34f7d2b2d48df18 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/5ccd8ab462434030b34f7d2b2d48df18 2024-12-04T19:15:21,352 DEBUG [StoreCloser-TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/41c356f67a9543d5ae9128d5d78580dd to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/41c356f67a9543d5ae9128d5d78580dd 2024-12-04T19:15:21,353 DEBUG [StoreCloser-TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/d45b55e78f644e5dabaf814d41f4e035 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/d45b55e78f644e5dabaf814d41f4e035 2024-12-04T19:15:21,354 DEBUG [StoreCloser-TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/27d763afc46340a5a5d4f541d357e7a0 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/27d763afc46340a5a5d4f541d357e7a0 2024-12-04T19:15:21,355 DEBUG [StoreCloser-TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/4ecc07e726d045dd8e95c474560b5d54 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/4ecc07e726d045dd8e95c474560b5d54 2024-12-04T19:15:21,356 DEBUG [StoreCloser-TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/21ea5931056f45d08b8348e5c21f67dd to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/21ea5931056f45d08b8348e5c21f67dd 2024-12-04T19:15:21,357 DEBUG [StoreCloser-TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/95e105114fcd4d758e6c13c6c2de5480 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/95e105114fcd4d758e6c13c6c2de5480 2024-12-04T19:15:21,358 DEBUG [StoreCloser-TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/b90855e589394347808aea074cc66c2f to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/b90855e589394347808aea074cc66c2f 2024-12-04T19:15:21,359 DEBUG [StoreCloser-TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/17a6d85075974b8e8a5b51fe16ab9c40 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/17a6d85075974b8e8a5b51fe16ab9c40 2024-12-04T19:15:21,360 DEBUG [StoreCloser-TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/14fa4989bbdb4ccc9b067726dac45d6f to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/14fa4989bbdb4ccc9b067726dac45d6f 2024-12-04T19:15:21,362 DEBUG [StoreCloser-TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/c1a884816070474cbd3a878ff3a6a32e to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/c1a884816070474cbd3a878ff3a6a32e 2024-12-04T19:15:21,363 DEBUG [StoreCloser-TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/fb281ae36a1949a1952ac2b5f45f7ea7 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/fb281ae36a1949a1952ac2b5f45f7ea7 2024-12-04T19:15:21,364 DEBUG [StoreCloser-TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/4488084553434cedb90010ec357e5a13 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/4488084553434cedb90010ec357e5a13 2024-12-04T19:15:21,366 DEBUG [StoreCloser-TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/7f3c4e3a33ce41aeb2b009b13c7f75e0 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/7f3c4e3a33ce41aeb2b009b13c7f75e0 2024-12-04T19:15:21,367 DEBUG [StoreCloser-TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/a0d776413f1d4bc38705cf93911376aa to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/a0d776413f1d4bc38705cf93911376aa 2024-12-04T19:15:21,368 DEBUG [StoreCloser-TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/872daeb46d904501be56a9dab8911cd8 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/872daeb46d904501be56a9dab8911cd8 2024-12-04T19:15:21,370 DEBUG [StoreCloser-TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/385292fa7a77471bb165da5bc1223c92 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/385292fa7a77471bb165da5bc1223c92 2024-12-04T19:15:21,371 DEBUG [StoreCloser-TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/4d467314615c437cb477a768bf4f222e to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/4d467314615c437cb477a768bf4f222e 2024-12-04T19:15:21,373 DEBUG [StoreCloser-TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/b3b753a3b80b488b9b3d9b715c9fc77f to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/b3b753a3b80b488b9b3d9b715c9fc77f 2024-12-04T19:15:21,374 DEBUG [StoreCloser-TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/61117ccc752e4b969e343b5a8b8df808 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/61117ccc752e4b969e343b5a8b8df808 2024-12-04T19:15:21,376 DEBUG [StoreCloser-TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/67ba12cacd1a48cbacc212b6f6d0e8c8 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/67ba12cacd1a48cbacc212b6f6d0e8c8 2024-12-04T19:15:21,378 DEBUG [StoreCloser-TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/ee93cd6bc3af4d7ca4522ff6a9335299 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/ee93cd6bc3af4d7ca4522ff6a9335299 2024-12-04T19:15:21,380 DEBUG [StoreCloser-TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/afee1509bf6a4606a100a5cd40a24a2a to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/afee1509bf6a4606a100a5cd40a24a2a 2024-12-04T19:15:21,381 DEBUG [StoreCloser-TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/566ca0b2462145d0a41d21e701ca4dea to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/566ca0b2462145d0a41d21e701ca4dea 2024-12-04T19:15:21,383 DEBUG [StoreCloser-TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/6001cbf0269f439399b6b6c426147783, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/31535e90f5cd40eb9507bad66eb08ce1, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/a0a65f53492e415281cd487ae9a8df53, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/8ad544a1e66a44bd88fcc357cdff4f66, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/d5929233b4b34345a5ba446e1a55d13e, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/eacc9abf7b5d407988a54784987a450d, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/252d2957b54c467fa12eff14525362ac, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/e62474a74061431295169a9ec26831c5, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/751b6a70966f4bd889ddbe6bd9525b5f, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/f984cbfc576d405b89ef6e583d849779, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/cad11e7bfe084c49821b94efbb8f45cc, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/f1798900b50a4327bad01ac424f9b357, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/0fcd19f034784ea1a4df54c21e6b5c44, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/233f006db5e34bcbb7cbd5da64df6b34, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/b025ed1d183b470aa1c50a7db15771fc, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/b5d0de0166b544f7ae1c0dc2b36a0ec1, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/4186a22a6dd1412faa24e75052b69f9b, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/66d74161b2c7444e8ac96c6224524518, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/c7bd652b08ed43c0bcaef7a8dda50fcb, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/67f11390b1ee4ba6947f6d4322a09d14, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/4b0cfdbd0f1e4e1eae3954e200218939, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/6592b52e005a460c8b880af047359665, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/6a8d0d242a5e46f48c01e09e85b3a48b, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/61ff286b02c34d24ae2793dd25092397, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/e43041abf46349a49601cfcc3f0e78f1, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/6acc9d08422b423ebac03db8b7e6b32a] to archive 2024-12-04T19:15:21,383 DEBUG [StoreCloser-TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-04T19:15:21,385 DEBUG [StoreCloser-TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/6001cbf0269f439399b6b6c426147783 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/6001cbf0269f439399b6b6c426147783 2024-12-04T19:15:21,386 DEBUG [StoreCloser-TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/31535e90f5cd40eb9507bad66eb08ce1 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/31535e90f5cd40eb9507bad66eb08ce1 2024-12-04T19:15:21,387 DEBUG [StoreCloser-TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/a0a65f53492e415281cd487ae9a8df53 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/a0a65f53492e415281cd487ae9a8df53 2024-12-04T19:15:21,388 DEBUG [StoreCloser-TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/8ad544a1e66a44bd88fcc357cdff4f66 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/8ad544a1e66a44bd88fcc357cdff4f66 2024-12-04T19:15:21,389 DEBUG [StoreCloser-TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/d5929233b4b34345a5ba446e1a55d13e to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/d5929233b4b34345a5ba446e1a55d13e 2024-12-04T19:15:21,390 DEBUG [StoreCloser-TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/eacc9abf7b5d407988a54784987a450d to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/eacc9abf7b5d407988a54784987a450d 2024-12-04T19:15:21,391 DEBUG [StoreCloser-TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/252d2957b54c467fa12eff14525362ac to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/252d2957b54c467fa12eff14525362ac 2024-12-04T19:15:21,392 DEBUG [StoreCloser-TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/e62474a74061431295169a9ec26831c5 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/e62474a74061431295169a9ec26831c5 2024-12-04T19:15:21,393 DEBUG [StoreCloser-TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/751b6a70966f4bd889ddbe6bd9525b5f to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/751b6a70966f4bd889ddbe6bd9525b5f 2024-12-04T19:15:21,394 DEBUG [StoreCloser-TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/f984cbfc576d405b89ef6e583d849779 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/f984cbfc576d405b89ef6e583d849779 2024-12-04T19:15:21,396 DEBUG [StoreCloser-TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/cad11e7bfe084c49821b94efbb8f45cc to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/cad11e7bfe084c49821b94efbb8f45cc 2024-12-04T19:15:21,397 DEBUG [StoreCloser-TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/f1798900b50a4327bad01ac424f9b357 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/f1798900b50a4327bad01ac424f9b357 2024-12-04T19:15:21,398 DEBUG [StoreCloser-TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/0fcd19f034784ea1a4df54c21e6b5c44 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/0fcd19f034784ea1a4df54c21e6b5c44 2024-12-04T19:15:21,399 DEBUG [StoreCloser-TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/233f006db5e34bcbb7cbd5da64df6b34 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/233f006db5e34bcbb7cbd5da64df6b34 2024-12-04T19:15:21,401 DEBUG [StoreCloser-TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/b025ed1d183b470aa1c50a7db15771fc to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/b025ed1d183b470aa1c50a7db15771fc 2024-12-04T19:15:21,402 DEBUG [StoreCloser-TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/b5d0de0166b544f7ae1c0dc2b36a0ec1 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/b5d0de0166b544f7ae1c0dc2b36a0ec1 2024-12-04T19:15:21,403 DEBUG [StoreCloser-TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/4186a22a6dd1412faa24e75052b69f9b to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/4186a22a6dd1412faa24e75052b69f9b 2024-12-04T19:15:21,405 DEBUG [StoreCloser-TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/66d74161b2c7444e8ac96c6224524518 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/66d74161b2c7444e8ac96c6224524518 2024-12-04T19:15:21,406 DEBUG [StoreCloser-TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/c7bd652b08ed43c0bcaef7a8dda50fcb to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/c7bd652b08ed43c0bcaef7a8dda50fcb 2024-12-04T19:15:21,408 DEBUG [StoreCloser-TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/67f11390b1ee4ba6947f6d4322a09d14 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/67f11390b1ee4ba6947f6d4322a09d14 2024-12-04T19:15:21,410 DEBUG [StoreCloser-TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/4b0cfdbd0f1e4e1eae3954e200218939 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/4b0cfdbd0f1e4e1eae3954e200218939 2024-12-04T19:15:21,411 DEBUG [StoreCloser-TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/6592b52e005a460c8b880af047359665 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/6592b52e005a460c8b880af047359665 2024-12-04T19:15:21,413 DEBUG [StoreCloser-TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/6a8d0d242a5e46f48c01e09e85b3a48b to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/6a8d0d242a5e46f48c01e09e85b3a48b 2024-12-04T19:15:21,415 DEBUG [StoreCloser-TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/61ff286b02c34d24ae2793dd25092397 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/61ff286b02c34d24ae2793dd25092397 2024-12-04T19:15:21,416 DEBUG [StoreCloser-TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/e43041abf46349a49601cfcc3f0e78f1 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/e43041abf46349a49601cfcc3f0e78f1 2024-12-04T19:15:21,418 DEBUG [StoreCloser-TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/6acc9d08422b423ebac03db8b7e6b32a to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/6acc9d08422b423ebac03db8b7e6b32a 2024-12-04T19:15:21,420 DEBUG [StoreCloser-TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/5d01b804b8504e3cbee56d8252ee14ee, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/6de385aac68a41ee84a839e1814a7292, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/afe53e5d74ec41d182bcd3d1ef0c60a8, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/9aaa03f052ef477c9ca586e06c9fb4ff, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/0beffc7e4b05492398e612bc8763d934, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/f126694142444ca3a70c7dc9e9890a63, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/e29f983fbbd24a19b76696ecca48b866, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/53e3c78d791a4358951dd2e96f5133a3, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/53175274a1614f489d50be4868316c5d, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/53ad0a4dc25b41d4b63ea31c77a7abde, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/e1c2b2983008489194812aa57aa006f1, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/9aab86884c584f3680fd09039b58808a, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/f86f0e4d90254cb1a88b430f5fa44385, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/a8d9868e17bb4debb8216351bd3b519b, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/0efc09240177462596022f413b802f61, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/8a143da60c644a44b451bef3fa6ad266, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/0d0b554e65b2422d8943336dc4002365, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/b0964ab1e88a45cfa5079a7a2d6a683f, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/3de4ad6ee4fb473ab6b2af742eeb2ac9, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/8c9123c4e5174c4ea6fd93f36c99afe5, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/45b6b3baebed41c3bf81c48d273e471f, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/170f889e88834407a206b138017d137f, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/196456b5a3de44b5b64e72365f966577] to archive 2024-12-04T19:15:21,421 DEBUG [StoreCloser-TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-04T19:15:21,422 DEBUG [StoreCloser-TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/5d01b804b8504e3cbee56d8252ee14ee to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/5d01b804b8504e3cbee56d8252ee14ee 2024-12-04T19:15:21,423 DEBUG [StoreCloser-TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/6de385aac68a41ee84a839e1814a7292 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/6de385aac68a41ee84a839e1814a7292 2024-12-04T19:15:21,424 DEBUG [StoreCloser-TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/afe53e5d74ec41d182bcd3d1ef0c60a8 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/afe53e5d74ec41d182bcd3d1ef0c60a8 2024-12-04T19:15:21,424 DEBUG [StoreCloser-TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/9aaa03f052ef477c9ca586e06c9fb4ff to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/9aaa03f052ef477c9ca586e06c9fb4ff 2024-12-04T19:15:21,425 DEBUG [StoreCloser-TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/0beffc7e4b05492398e612bc8763d934 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/0beffc7e4b05492398e612bc8763d934 2024-12-04T19:15:21,426 DEBUG [StoreCloser-TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/f126694142444ca3a70c7dc9e9890a63 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/f126694142444ca3a70c7dc9e9890a63 2024-12-04T19:15:21,426 DEBUG [StoreCloser-TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/e29f983fbbd24a19b76696ecca48b866 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/e29f983fbbd24a19b76696ecca48b866 2024-12-04T19:15:21,427 DEBUG [StoreCloser-TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/53e3c78d791a4358951dd2e96f5133a3 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/53e3c78d791a4358951dd2e96f5133a3 2024-12-04T19:15:21,428 DEBUG [StoreCloser-TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/53175274a1614f489d50be4868316c5d to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/53175274a1614f489d50be4868316c5d 2024-12-04T19:15:21,428 DEBUG [StoreCloser-TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/53ad0a4dc25b41d4b63ea31c77a7abde to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/53ad0a4dc25b41d4b63ea31c77a7abde 2024-12-04T19:15:21,429 DEBUG [StoreCloser-TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/e1c2b2983008489194812aa57aa006f1 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/e1c2b2983008489194812aa57aa006f1 2024-12-04T19:15:21,430 DEBUG [StoreCloser-TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/9aab86884c584f3680fd09039b58808a to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/9aab86884c584f3680fd09039b58808a 2024-12-04T19:15:21,430 DEBUG [StoreCloser-TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/f86f0e4d90254cb1a88b430f5fa44385 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/f86f0e4d90254cb1a88b430f5fa44385 2024-12-04T19:15:21,431 DEBUG [StoreCloser-TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/a8d9868e17bb4debb8216351bd3b519b to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/a8d9868e17bb4debb8216351bd3b519b 2024-12-04T19:15:21,431 DEBUG [StoreCloser-TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/0efc09240177462596022f413b802f61 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/0efc09240177462596022f413b802f61 2024-12-04T19:15:21,432 DEBUG [StoreCloser-TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/8a143da60c644a44b451bef3fa6ad266 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/8a143da60c644a44b451bef3fa6ad266 2024-12-04T19:15:21,433 DEBUG [StoreCloser-TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/0d0b554e65b2422d8943336dc4002365 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/0d0b554e65b2422d8943336dc4002365 2024-12-04T19:15:21,434 DEBUG [StoreCloser-TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/b0964ab1e88a45cfa5079a7a2d6a683f to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/b0964ab1e88a45cfa5079a7a2d6a683f 2024-12-04T19:15:21,434 DEBUG [StoreCloser-TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/3de4ad6ee4fb473ab6b2af742eeb2ac9 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/3de4ad6ee4fb473ab6b2af742eeb2ac9 2024-12-04T19:15:21,435 DEBUG [StoreCloser-TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/8c9123c4e5174c4ea6fd93f36c99afe5 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/8c9123c4e5174c4ea6fd93f36c99afe5 2024-12-04T19:15:21,436 DEBUG [StoreCloser-TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/45b6b3baebed41c3bf81c48d273e471f to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/45b6b3baebed41c3bf81c48d273e471f 2024-12-04T19:15:21,437 DEBUG [StoreCloser-TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/170f889e88834407a206b138017d137f to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/170f889e88834407a206b138017d137f 2024-12-04T19:15:21,437 DEBUG [StoreCloser-TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/196456b5a3de44b5b64e72365f966577 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/196456b5a3de44b5b64e72365f966577 2024-12-04T19:15:21,441 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/recovered.edits/482.seqid, newMaxSeqId=482, maxSeqId=1 2024-12-04T19:15:21,442 INFO [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. 2024-12-04T19:15:21,442 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1676): Region close journal for 6c5c1603a070441ac38abd9294c86a68: Waiting for close lock at 1733339719914Running coprocessor pre-close hooks at 1733339719914Disabling compacts and flushes for region at 1733339719914Disabling writes for close at 1733339720077 (+163 ms)Obtaining lock to block concurrent updates at 1733339720077Preparing flush snapshotting stores in 6c5c1603a070441ac38abd9294c86a68 at 1733339720077Finished memstore snapshotting TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1733339720078 (+1 ms)Flushing stores of TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68. at 1733339720078Flushing 6c5c1603a070441ac38abd9294c86a68/A: creating writer at 1733339720078Flushing 6c5c1603a070441ac38abd9294c86a68/A: appending metadata at 1733339720081 (+3 ms)Flushing 6c5c1603a070441ac38abd9294c86a68/A: closing flushed file at 1733339720081Flushing 6c5c1603a070441ac38abd9294c86a68/B: creating writer at 1733339720496 (+415 ms)Flushing 6c5c1603a070441ac38abd9294c86a68/B: appending metadata at 1733339720500 (+4 ms)Flushing 6c5c1603a070441ac38abd9294c86a68/B: closing flushed file at 1733339720500Flushing 6c5c1603a070441ac38abd9294c86a68/C: creating writer at 1733339720914 (+414 ms)Flushing 6c5c1603a070441ac38abd9294c86a68/C: appending metadata at 1733339720918 (+4 ms)Flushing 6c5c1603a070441ac38abd9294c86a68/C: closing flushed file at 1733339720918Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@761ce5dd: reopening flushed file at 1733339721329 (+411 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@c5330bf: reopening flushed file at 1733339721336 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@61561b41: reopening flushed file at 1733339721341 (+5 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 6c5c1603a070441ac38abd9294c86a68 in 1268ms, sequenceid=479, compaction requested=true at 1733339721346 (+5 ms)Writing region close event to WAL at 1733339721439 (+93 ms)Running coprocessor post-close hooks at 1733339721441 (+2 ms)Closed at 1733339721442 (+1 ms) 2024-12-04T19:15:21,443 INFO [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(157): Closed 6c5c1603a070441ac38abd9294c86a68 2024-12-04T19:15:21,444 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=6c5c1603a070441ac38abd9294c86a68, regionState=CLOSED 2024-12-04T19:15:21,445 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=85, ppid=84, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 because future has completed 2024-12-04T19:15:21,447 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-12-04T19:15:21,447 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; CloseRegionProcedure 6c5c1603a070441ac38abd9294c86a68, server=9ba2734cd39f,33969,1733339629791 in 1.6850 sec 2024-12-04T19:15:21,450 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=84, resume processing ppid=83 2024-12-04T19:15:21,450 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, ppid=83, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6c5c1603a070441ac38abd9294c86a68, UNASSIGN in 1.6890 sec 2024-12-04T19:15:21,452 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-12-04T19:15:21,452 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.6920 sec 2024-12-04T19:15:21,454 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733339721454"}]},"ts":"1733339721454"} 2024-12-04T19:15:21,456 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-04T19:15:21,456 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-04T19:15:21,458 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.7020 sec 2024-12-04T19:15:21,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-04T19:15:21,903 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-04T19:15:21,904 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-04T19:15:21,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] procedure2.ProcedureExecutor(1139): Stored pid=86, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T19:15:21,907 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=86, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T19:15:21,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-12-04T19:15:21,909 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=86, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T19:15:21,912 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68 2024-12-04T19:15:21,915 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A, FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B, FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C, FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/recovered.edits] 2024-12-04T19:15:21,918 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/11015cd8073944008663bb4cb2378022 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/11015cd8073944008663bb4cb2378022 2024-12-04T19:15:21,919 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/ba1e994b9121457297b8cb48a85bf750 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/A/ba1e994b9121457297b8cb48a85bf750 2024-12-04T19:15:21,921 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/386e5842914d4c8abcd910bd0d0644e6 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/386e5842914d4c8abcd910bd0d0644e6 2024-12-04T19:15:21,922 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/674dcf059dca4b9f8176c0fec525654a to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/B/674dcf059dca4b9f8176c0fec525654a 2024-12-04T19:15:21,924 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/22153514dffd4ab5ab4f1184cc89b39d to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/22153514dffd4ab5ab4f1184cc89b39d 2024-12-04T19:15:21,925 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/80eb6b87beec4bc28682060f42bde326 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/80eb6b87beec4bc28682060f42bde326 2024-12-04T19:15:21,926 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/acbb3fb7b7f8419483297cda88fd0470 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/acbb3fb7b7f8419483297cda88fd0470 2024-12-04T19:15:21,927 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/f57f3e15f4b244328fa6f982eef3321c to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/C/f57f3e15f4b244328fa6f982eef3321c 2024-12-04T19:15:21,930 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/recovered.edits/482.seqid to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68/recovered.edits/482.seqid 2024-12-04T19:15:21,931 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/6c5c1603a070441ac38abd9294c86a68 2024-12-04T19:15:21,931 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-04T19:15:21,933 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=86, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T19:15:21,937 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-04T19:15:21,940 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-04T19:15:21,942 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=86, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T19:15:21,942 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-04T19:15:21,942 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733339721942"}]},"ts":"9223372036854775807"} 2024-12-04T19:15:21,945 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-04T19:15:21,945 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 6c5c1603a070441ac38abd9294c86a68, NAME => 'TestAcidGuarantees,,1733339696000.6c5c1603a070441ac38abd9294c86a68.', STARTKEY => '', ENDKEY => ''}] 2024-12-04T19:15:21,945 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-04T19:15:21,945 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733339721945"}]},"ts":"9223372036854775807"} 2024-12-04T19:15:21,947 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-04T19:15:21,947 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=86, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T19:15:21,948 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 43 msec 2024-12-04T19:15:22,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-12-04T19:15:22,012 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-04T19:15:22,012 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-04T19:15:22,025 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=250 (was 252), OpenFileDescriptor=453 (was 459), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=456 (was 429) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=14920 (was 15034) 2024-12-04T19:15:22,034 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=250, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=456, ProcessCount=11, AvailableMemoryMB=14920 2024-12-04T19:15:22,036 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-04T19:15:22,036 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T19:15:22,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] procedure2.ProcedureExecutor(1139): Stored pid=87, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-04T19:15:22,038 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-04T19:15:22,038 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:22,038 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 87 2024-12-04T19:15:22,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-12-04T19:15:22,039 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-04T19:15:22,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742163_1339 (size=1061) 2024-12-04T19:15:22,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-12-04T19:15:22,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-12-04T19:15:22,450 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d 2024-12-04T19:15:22,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742164_1340 (size=53) 2024-12-04T19:15:22,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-12-04T19:15:22,860 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T19:15:22,860 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 77154c9b50d6b1163c1785212a1cb410, disabling compactions & flushes 2024-12-04T19:15:22,860 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:22,860 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:22,860 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. after waiting 0 ms 2024-12-04T19:15:22,860 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:22,860 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:22,860 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 77154c9b50d6b1163c1785212a1cb410: Waiting for close lock at 1733339722860Disabling compacts and flushes for region at 1733339722860Disabling writes for close at 1733339722860Writing region close event to WAL at 1733339722860Closed at 1733339722860 2024-12-04T19:15:22,862 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-04T19:15:22,862 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733339722862"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733339722862"}]},"ts":"1733339722862"} 2024-12-04T19:15:22,864 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-04T19:15:22,865 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-04T19:15:22,865 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733339722865"}]},"ts":"1733339722865"} 2024-12-04T19:15:22,866 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-04T19:15:22,867 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=77154c9b50d6b1163c1785212a1cb410, ASSIGN}] 2024-12-04T19:15:22,868 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=77154c9b50d6b1163c1785212a1cb410, ASSIGN 2024-12-04T19:15:22,869 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=77154c9b50d6b1163c1785212a1cb410, ASSIGN; state=OFFLINE, location=9ba2734cd39f,33969,1733339629791; forceNewPlan=false, retain=false 2024-12-04T19:15:23,020 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=77154c9b50d6b1163c1785212a1cb410, regionState=OPENING, regionLocation=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:23,024 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=77154c9b50d6b1163c1785212a1cb410, ASSIGN because future has completed 2024-12-04T19:15:23,024 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE, hasLock=false; OpenRegionProcedure 77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791}] 2024-12-04T19:15:23,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-12-04T19:15:23,180 INFO [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:23,180 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(7752): Opening region: {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} 2024-12-04T19:15:23,180 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 77154c9b50d6b1163c1785212a1cb410 2024-12-04T19:15:23,180 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T19:15:23,180 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(7794): checking encryption for 77154c9b50d6b1163c1785212a1cb410 2024-12-04T19:15:23,180 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(7797): checking classloading for 77154c9b50d6b1163c1785212a1cb410 2024-12-04T19:15:23,182 INFO [StoreOpener-77154c9b50d6b1163c1785212a1cb410-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 77154c9b50d6b1163c1785212a1cb410 2024-12-04T19:15:23,184 INFO [StoreOpener-77154c9b50d6b1163c1785212a1cb410-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-04T19:15:23,184 INFO [StoreOpener-77154c9b50d6b1163c1785212a1cb410-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 77154c9b50d6b1163c1785212a1cb410 columnFamilyName A 2024-12-04T19:15:23,184 DEBUG [StoreOpener-77154c9b50d6b1163c1785212a1cb410-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:23,185 INFO [StoreOpener-77154c9b50d6b1163c1785212a1cb410-1 {}] regionserver.HStore(327): Store=77154c9b50d6b1163c1785212a1cb410/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:15:23,185 INFO [StoreOpener-77154c9b50d6b1163c1785212a1cb410-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 77154c9b50d6b1163c1785212a1cb410 2024-12-04T19:15:23,186 INFO [StoreOpener-77154c9b50d6b1163c1785212a1cb410-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-04T19:15:23,186 INFO [StoreOpener-77154c9b50d6b1163c1785212a1cb410-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 77154c9b50d6b1163c1785212a1cb410 columnFamilyName B 2024-12-04T19:15:23,186 DEBUG [StoreOpener-77154c9b50d6b1163c1785212a1cb410-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:23,187 INFO [StoreOpener-77154c9b50d6b1163c1785212a1cb410-1 {}] regionserver.HStore(327): Store=77154c9b50d6b1163c1785212a1cb410/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:15:23,187 INFO [StoreOpener-77154c9b50d6b1163c1785212a1cb410-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 77154c9b50d6b1163c1785212a1cb410 2024-12-04T19:15:23,188 INFO [StoreOpener-77154c9b50d6b1163c1785212a1cb410-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-04T19:15:23,188 INFO [StoreOpener-77154c9b50d6b1163c1785212a1cb410-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 77154c9b50d6b1163c1785212a1cb410 columnFamilyName C 2024-12-04T19:15:23,188 DEBUG [StoreOpener-77154c9b50d6b1163c1785212a1cb410-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:23,189 INFO [StoreOpener-77154c9b50d6b1163c1785212a1cb410-1 {}] regionserver.HStore(327): Store=77154c9b50d6b1163c1785212a1cb410/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:15:23,189 INFO [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:23,189 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1038): replaying wal for 77154c9b50d6b1163c1785212a1cb410 2024-12-04T19:15:23,189 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410 2024-12-04T19:15:23,190 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410 2024-12-04T19:15:23,190 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1048): stopping wal replay for 77154c9b50d6b1163c1785212a1cb410 2024-12-04T19:15:23,190 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1060): Cleaning up temporary data for 77154c9b50d6b1163c1785212a1cb410 2024-12-04T19:15:23,191 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-04T19:15:23,192 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1093): writing seq id for 77154c9b50d6b1163c1785212a1cb410 2024-12-04T19:15:23,193 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T19:15:23,194 INFO [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1114): Opened 77154c9b50d6b1163c1785212a1cb410; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59910898, jitterRate=-0.10725805163383484}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-04T19:15:23,194 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 77154c9b50d6b1163c1785212a1cb410 2024-12-04T19:15:23,194 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1006): Region open journal for 77154c9b50d6b1163c1785212a1cb410: Running coprocessor pre-open hook at 1733339723181Writing region info on filesystem at 1733339723181Initializing all the Stores at 1733339723181Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733339723182 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733339723182Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733339723182Cleaning up temporary data from old regions at 1733339723190 (+8 ms)Running coprocessor post-open hooks at 1733339723194 (+4 ms)Region opened successfully at 1733339723194 2024-12-04T19:15:23,195 INFO [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., pid=89, masterSystemTime=1733339723176 2024-12-04T19:15:23,196 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:23,196 INFO [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:23,197 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=77154c9b50d6b1163c1785212a1cb410, regionState=OPEN, openSeqNum=2, regionLocation=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:23,198 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=89, ppid=88, state=RUNNABLE, hasLock=false; OpenRegionProcedure 77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 because future has completed 2024-12-04T19:15:23,200 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-12-04T19:15:23,200 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; OpenRegionProcedure 77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 in 174 msec 2024-12-04T19:15:23,201 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=87 2024-12-04T19:15:23,202 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=87, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=77154c9b50d6b1163c1785212a1cb410, ASSIGN in 333 msec 2024-12-04T19:15:23,202 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-04T19:15:23,202 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733339723202"}]},"ts":"1733339723202"} 2024-12-04T19:15:23,203 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-04T19:15:23,204 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-04T19:15:23,205 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1680 sec 2024-12-04T19:15:24,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-12-04T19:15:24,182 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-04T19:15:24,184 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7789cf32, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:15:24,184 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ba2734cd39f,46667,-1 for getting cluster id 2024-12-04T19:15:24,184 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T19:15:24,185 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9baaea35-8dd3-4653-9b7c-03e5b792e644' 2024-12-04T19:15:24,186 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T19:15:24,186 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9baaea35-8dd3-4653-9b7c-03e5b792e644" 2024-12-04T19:15:24,186 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2eb0d622, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:15:24,186 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ba2734cd39f,46667,-1] 2024-12-04T19:15:24,186 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T19:15:24,187 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:15:24,188 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41672, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T19:15:24,188 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f790636, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:15:24,189 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T19:15:24,190 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ba2734cd39f,33969,1733339629791, seqNum=-1] 2024-12-04T19:15:24,190 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T19:15:24,191 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49032, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T19:15:24,192 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-04T19:15:24,193 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 9ba2734cd39f,46667,1733339628877 2024-12-04T19:15:24,193 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@68460717 2024-12-04T19:15:24,193 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-04T19:15:24,194 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41676, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-04T19:15:24,196 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-04T19:15:24,196 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T19:15:24,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] procedure2.ProcedureExecutor(1139): Stored pid=90, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-04T19:15:24,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742165_1341 (size=1097) 2024-12-04T19:15:24,612 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-04T19:15:24,613 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-04T19:15:24,618 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-04T19:15:24,621 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=77154c9b50d6b1163c1785212a1cb410, REOPEN/MOVE}] 2024-12-04T19:15:24,623 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=77154c9b50d6b1163c1785212a1cb410, REOPEN/MOVE 2024-12-04T19:15:24,624 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=77154c9b50d6b1163c1785212a1cb410, regionState=CLOSING, regionLocation=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:24,626 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=77154c9b50d6b1163c1785212a1cb410, REOPEN/MOVE because future has completed 2024-12-04T19:15:24,627 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-04T19:15:24,627 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=93, ppid=92, state=RUNNABLE, hasLock=false; CloseRegionProcedure 77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791}] 2024-12-04T19:15:24,781 INFO [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] handler.UnassignRegionHandler(122): Close 77154c9b50d6b1163c1785212a1cb410 2024-12-04T19:15:24,781 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-04T19:15:24,781 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1722): Closing 77154c9b50d6b1163c1785212a1cb410, disabling compactions & flushes 2024-12-04T19:15:24,781 INFO [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:24,781 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:24,781 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. after waiting 0 ms 2024-12-04T19:15:24,781 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:24,789 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-04T19:15:24,790 INFO [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:24,791 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1676): Region close journal for 77154c9b50d6b1163c1785212a1cb410: Waiting for close lock at 1733339724781Running coprocessor pre-close hooks at 1733339724781Disabling compacts and flushes for region at 1733339724781Disabling writes for close at 1733339724781Writing region close event to WAL at 1733339724782 (+1 ms)Running coprocessor post-close hooks at 1733339724790 (+8 ms)Closed at 1733339724790 2024-12-04T19:15:24,791 WARN [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegionServer(3299): Not adding moved region record: 77154c9b50d6b1163c1785212a1cb410 to self. 2024-12-04T19:15:24,793 INFO [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] handler.UnassignRegionHandler(157): Closed 77154c9b50d6b1163c1785212a1cb410 2024-12-04T19:15:24,793 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=77154c9b50d6b1163c1785212a1cb410, regionState=CLOSED 2024-12-04T19:15:24,795 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=93, ppid=92, state=RUNNABLE, hasLock=false; CloseRegionProcedure 77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 because future has completed 2024-12-04T19:15:24,798 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=93, resume processing ppid=92 2024-12-04T19:15:24,798 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, ppid=92, state=SUCCESS, hasLock=false; CloseRegionProcedure 77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 in 169 msec 2024-12-04T19:15:24,799 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=77154c9b50d6b1163c1785212a1cb410, REOPEN/MOVE; state=CLOSED, location=9ba2734cd39f,33969,1733339629791; forceNewPlan=false, retain=true 2024-12-04T19:15:24,950 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=77154c9b50d6b1163c1785212a1cb410, regionState=OPENING, regionLocation=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:24,954 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=77154c9b50d6b1163c1785212a1cb410, REOPEN/MOVE because future has completed 2024-12-04T19:15:24,955 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=92, state=RUNNABLE, hasLock=false; OpenRegionProcedure 77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791}] 2024-12-04T19:15:25,116 INFO [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:25,117 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(7752): Opening region: {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} 2024-12-04T19:15:25,118 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 77154c9b50d6b1163c1785212a1cb410 2024-12-04T19:15:25,118 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T19:15:25,118 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(7794): checking encryption for 77154c9b50d6b1163c1785212a1cb410 2024-12-04T19:15:25,118 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(7797): checking classloading for 77154c9b50d6b1163c1785212a1cb410 2024-12-04T19:15:25,120 INFO [StoreOpener-77154c9b50d6b1163c1785212a1cb410-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 77154c9b50d6b1163c1785212a1cb410 2024-12-04T19:15:25,122 INFO [StoreOpener-77154c9b50d6b1163c1785212a1cb410-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-04T19:15:25,122 INFO [StoreOpener-77154c9b50d6b1163c1785212a1cb410-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 77154c9b50d6b1163c1785212a1cb410 columnFamilyName A 2024-12-04T19:15:25,124 DEBUG [StoreOpener-77154c9b50d6b1163c1785212a1cb410-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:25,125 INFO [StoreOpener-77154c9b50d6b1163c1785212a1cb410-1 {}] regionserver.HStore(327): Store=77154c9b50d6b1163c1785212a1cb410/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:15:25,125 INFO [StoreOpener-77154c9b50d6b1163c1785212a1cb410-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 77154c9b50d6b1163c1785212a1cb410 2024-12-04T19:15:25,126 INFO [StoreOpener-77154c9b50d6b1163c1785212a1cb410-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-04T19:15:25,127 INFO [StoreOpener-77154c9b50d6b1163c1785212a1cb410-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 77154c9b50d6b1163c1785212a1cb410 columnFamilyName B 2024-12-04T19:15:25,127 DEBUG [StoreOpener-77154c9b50d6b1163c1785212a1cb410-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:25,128 INFO [StoreOpener-77154c9b50d6b1163c1785212a1cb410-1 {}] regionserver.HStore(327): Store=77154c9b50d6b1163c1785212a1cb410/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:15:25,128 INFO [StoreOpener-77154c9b50d6b1163c1785212a1cb410-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 77154c9b50d6b1163c1785212a1cb410 2024-12-04T19:15:25,129 INFO [StoreOpener-77154c9b50d6b1163c1785212a1cb410-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-04T19:15:25,129 INFO [StoreOpener-77154c9b50d6b1163c1785212a1cb410-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 77154c9b50d6b1163c1785212a1cb410 columnFamilyName C 2024-12-04T19:15:25,129 DEBUG [StoreOpener-77154c9b50d6b1163c1785212a1cb410-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:25,130 INFO [StoreOpener-77154c9b50d6b1163c1785212a1cb410-1 {}] regionserver.HStore(327): Store=77154c9b50d6b1163c1785212a1cb410/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:15:25,130 INFO [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:25,130 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1038): replaying wal for 77154c9b50d6b1163c1785212a1cb410 2024-12-04T19:15:25,131 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410 2024-12-04T19:15:25,133 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410 2024-12-04T19:15:25,135 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1048): stopping wal replay for 77154c9b50d6b1163c1785212a1cb410 2024-12-04T19:15:25,135 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1060): Cleaning up temporary data for 77154c9b50d6b1163c1785212a1cb410 2024-12-04T19:15:25,135 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-04T19:15:25,137 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1093): writing seq id for 77154c9b50d6b1163c1785212a1cb410 2024-12-04T19:15:25,138 INFO [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1114): Opened 77154c9b50d6b1163c1785212a1cb410; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68633627, jitterRate=0.022720739245414734}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-04T19:15:25,138 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 77154c9b50d6b1163c1785212a1cb410 2024-12-04T19:15:25,139 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1006): Region open journal for 77154c9b50d6b1163c1785212a1cb410: Running coprocessor pre-open hook at 1733339725118Writing region info on filesystem at 1733339725118Initializing all the Stores at 1733339725120 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733339725120Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733339725120Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733339725120Cleaning up temporary data from old regions at 1733339725135 (+15 ms)Running coprocessor post-open hooks at 1733339725138 (+3 ms)Region opened successfully at 1733339725139 (+1 ms) 2024-12-04T19:15:25,140 INFO [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., pid=94, masterSystemTime=1733339725109 2024-12-04T19:15:25,142 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:25,142 INFO [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:25,142 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=77154c9b50d6b1163c1785212a1cb410, regionState=OPEN, openSeqNum=5, regionLocation=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:25,144 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=94, ppid=92, state=RUNNABLE, hasLock=false; OpenRegionProcedure 77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 because future has completed 2024-12-04T19:15:25,146 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=92 2024-12-04T19:15:25,146 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=92, state=SUCCESS, hasLock=false; OpenRegionProcedure 77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 in 189 msec 2024-12-04T19:15:25,148 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-12-04T19:15:25,148 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=77154c9b50d6b1163c1785212a1cb410, REOPEN/MOVE in 525 msec 2024-12-04T19:15:25,150 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-12-04T19:15:25,150 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 531 msec 2024-12-04T19:15:25,152 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 954 msec 2024-12-04T19:15:25,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-12-04T19:15:25,153 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-04T19:15:25,153 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28a884f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:15:25,153 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ba2734cd39f,46667,-1 for getting cluster id 2024-12-04T19:15:25,153 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T19:15:25,154 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9baaea35-8dd3-4653-9b7c-03e5b792e644' 2024-12-04T19:15:25,154 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T19:15:25,154 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9baaea35-8dd3-4653-9b7c-03e5b792e644" 2024-12-04T19:15:25,154 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2dfb8c30, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:15:25,154 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ba2734cd39f,46667,-1] 2024-12-04T19:15:25,154 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T19:15:25,155 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:15:25,155 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41698, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T19:15:25,155 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44209046, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:15:25,156 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14f86ed2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:15:25,156 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ba2734cd39f,46667,-1 for getting cluster id 2024-12-04T19:15:25,156 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T19:15:25,156 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9baaea35-8dd3-4653-9b7c-03e5b792e644' 2024-12-04T19:15:25,156 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T19:15:25,157 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9baaea35-8dd3-4653-9b7c-03e5b792e644" 2024-12-04T19:15:25,157 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a4f89aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:15:25,157 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ba2734cd39f,46667,-1] 2024-12-04T19:15:25,157 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T19:15:25,157 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:15:25,157 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41722, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T19:15:25,158 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@446d38b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:15:25,158 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59f0df0b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:15:25,158 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ba2734cd39f,46667,-1 for getting cluster id 2024-12-04T19:15:25,158 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T19:15:25,159 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9baaea35-8dd3-4653-9b7c-03e5b792e644' 2024-12-04T19:15:25,159 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T19:15:25,159 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9baaea35-8dd3-4653-9b7c-03e5b792e644" 2024-12-04T19:15:25,159 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b3bc9e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:15:25,159 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ba2734cd39f,46667,-1] 2024-12-04T19:15:25,159 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T19:15:25,159 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:15:25,160 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41738, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T19:15:25,160 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@582fc7a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:15:25,161 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@927442d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:15:25,161 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ba2734cd39f,46667,-1 for getting cluster id 2024-12-04T19:15:25,161 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T19:15:25,161 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9baaea35-8dd3-4653-9b7c-03e5b792e644' 2024-12-04T19:15:25,161 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T19:15:25,161 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9baaea35-8dd3-4653-9b7c-03e5b792e644" 2024-12-04T19:15:25,161 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@206c86ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:15:25,161 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ba2734cd39f,46667,-1] 2024-12-04T19:15:25,162 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T19:15:25,162 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:15:25,162 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41758, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T19:15:25,163 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ed28e95, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:15:25,163 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@725928f6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:15:25,163 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ba2734cd39f,46667,-1 for getting cluster id 2024-12-04T19:15:25,163 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T19:15:25,164 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9baaea35-8dd3-4653-9b7c-03e5b792e644' 2024-12-04T19:15:25,164 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T19:15:25,164 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9baaea35-8dd3-4653-9b7c-03e5b792e644" 2024-12-04T19:15:25,164 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3047a6e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:15:25,164 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ba2734cd39f,46667,-1] 2024-12-04T19:15:25,164 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T19:15:25,164 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:15:25,165 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41790, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T19:15:25,165 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@dc665d8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:15:25,166 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@227a416f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:15:25,166 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ba2734cd39f,46667,-1 for getting cluster id 2024-12-04T19:15:25,166 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T19:15:25,166 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9baaea35-8dd3-4653-9b7c-03e5b792e644' 2024-12-04T19:15:25,166 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T19:15:25,166 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9baaea35-8dd3-4653-9b7c-03e5b792e644" 2024-12-04T19:15:25,166 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4da4125, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:15:25,166 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ba2734cd39f,46667,-1] 2024-12-04T19:15:25,167 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T19:15:25,167 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:15:25,167 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41808, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T19:15:25,168 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d9289cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:15:25,168 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@580773ea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:15:25,168 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ba2734cd39f,46667,-1 for getting cluster id 2024-12-04T19:15:25,168 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T19:15:25,169 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9baaea35-8dd3-4653-9b7c-03e5b792e644' 2024-12-04T19:15:25,169 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T19:15:25,169 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9baaea35-8dd3-4653-9b7c-03e5b792e644" 2024-12-04T19:15:25,169 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@216e1587, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:15:25,169 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ba2734cd39f,46667,-1] 2024-12-04T19:15:25,169 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T19:15:25,169 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:15:25,170 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41824, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T19:15:25,170 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ed7d70d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:15:25,171 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34dcef37, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:15:25,171 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ba2734cd39f,46667,-1 for getting cluster id 2024-12-04T19:15:25,171 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T19:15:25,172 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9baaea35-8dd3-4653-9b7c-03e5b792e644' 2024-12-04T19:15:25,172 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T19:15:25,172 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9baaea35-8dd3-4653-9b7c-03e5b792e644" 2024-12-04T19:15:25,172 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fea4cb1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:15:25,173 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ba2734cd39f,46667,-1] 2024-12-04T19:15:25,173 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T19:15:25,173 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:15:25,173 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41840, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T19:15:25,174 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40c517d5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:15:25,175 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d5713b4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:15:25,175 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ba2734cd39f,46667,-1 for getting cluster id 2024-12-04T19:15:25,175 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T19:15:25,175 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9baaea35-8dd3-4653-9b7c-03e5b792e644' 2024-12-04T19:15:25,176 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T19:15:25,176 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9baaea35-8dd3-4653-9b7c-03e5b792e644" 2024-12-04T19:15:25,176 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@531087c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:15:25,176 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ba2734cd39f,46667,-1] 2024-12-04T19:15:25,176 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T19:15:25,176 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:15:25,177 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41854, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T19:15:25,177 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d2e7892, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:15:25,178 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d2483a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:15:25,178 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ba2734cd39f,46667,-1 for getting cluster id 2024-12-04T19:15:25,178 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T19:15:25,179 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9baaea35-8dd3-4653-9b7c-03e5b792e644' 2024-12-04T19:15:25,179 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T19:15:25,179 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9baaea35-8dd3-4653-9b7c-03e5b792e644" 2024-12-04T19:15:25,179 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77d2d55f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:15:25,179 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ba2734cd39f,46667,-1] 2024-12-04T19:15:25,180 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T19:15:25,180 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:15:25,180 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41870, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T19:15:25,181 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d8f851a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:15:25,181 DEBUG [Thread-1557 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T19:15:25,181 DEBUG [Thread-1570 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T19:15:25,181 DEBUG [Thread-1562 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T19:15:25,181 DEBUG [Thread-1559 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T19:15:25,182 DEBUG [Thread-1566 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T19:15:25,182 DEBUG [Thread-1568 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T19:15:25,182 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ba2734cd39f,33969,1733339629791, seqNum=-1] 2024-12-04T19:15:25,182 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ba2734cd39f,33969,1733339629791, seqNum=-1] 2024-12-04T19:15:25,182 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ba2734cd39f,33969,1733339629791, seqNum=-1] 2024-12-04T19:15:25,182 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ba2734cd39f,33969,1733339629791, seqNum=-1] 2024-12-04T19:15:25,182 DEBUG [Thread-1564 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T19:15:25,183 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T19:15:25,183 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T19:15:25,183 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T19:15:25,183 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T19:15:25,183 DEBUG [Thread-1555 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T19:15:25,184 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ba2734cd39f,33969,1733339629791, seqNum=-1] 2024-12-04T19:15:25,184 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ba2734cd39f,33969,1733339629791, seqNum=-1] 2024-12-04T19:15:25,184 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T19:15:25,184 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T19:15:25,184 DEBUG [Thread-1551 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T19:15:25,184 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49042, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T19:15:25,184 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49046, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T19:15:25,184 DEBUG [Thread-1553 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T19:15:25,185 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ba2734cd39f,33969,1733339629791, seqNum=-1] 2024-12-04T19:15:25,185 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ba2734cd39f,33969,1733339629791, seqNum=-1] 2024-12-04T19:15:25,185 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49044, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T19:15:25,185 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T19:15:25,185 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T19:15:25,185 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49070, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T19:15:25,185 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49102, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T19:15:25,185 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49094, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T19:15:25,185 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ba2734cd39f,33969,1733339629791, seqNum=-1] 2024-12-04T19:15:25,186 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T19:15:25,186 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ba2734cd39f,33969,1733339629791, seqNum=-1] 2024-12-04T19:15:25,186 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T19:15:25,186 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49084, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T19:15:25,186 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T19:15:25,186 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49118, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T19:15:25,186 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49134, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T19:15:25,187 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5] 2024-12-04T19:15:25,187 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5] 2024-12-04T19:15:25,187 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49148, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T19:15:25,187 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5] 2024-12-04T19:15:25,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees 2024-12-04T19:15:25,189 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5] 2024-12-04T19:15:25,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-04T19:15:25,190 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T19:15:25,190 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5] 2024-12-04T19:15:25,191 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5] 2024-12-04T19:15:25,191 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5] 2024-12-04T19:15:25,191 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5] 2024-12-04T19:15:25,192 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5] 2024-12-04T19:15:25,192 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T19:15:25,192 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5] 2024-12-04T19:15:25,192 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T19:15:25,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(8855): Flush requested on 77154c9b50d6b1163c1785212a1cb410 2024-12-04T19:15:25,200 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 77154c9b50d6b1163c1785212a1cb410 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-04T19:15:25,202 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77154c9b50d6b1163c1785212a1cb410, store=A 2024-12-04T19:15:25,202 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:15:25,202 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77154c9b50d6b1163c1785212a1cb410, store=B 2024-12-04T19:15:25,202 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:15:25,202 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77154c9b50d6b1163c1785212a1cb410, store=C 2024-12-04T19:15:25,202 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:15:25,247 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412042750cc672a3745c4a0c78e848f38ad1b_77154c9b50d6b1163c1785212a1cb410 is 50, key is test_row_0/A:col10/1733339725197/Put/seqid=0 2024-12-04T19:15:25,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742166_1342 (size=12154) 2024-12-04T19:15:25,252 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:25,255 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412042750cc672a3745c4a0c78e848f38ad1b_77154c9b50d6b1163c1785212a1cb410 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412042750cc672a3745c4a0c78e848f38ad1b_77154c9b50d6b1163c1785212a1cb410 2024-12-04T19:15:25,256 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/A/461d009f76954422b02d34e1df1615ea, store: [table=TestAcidGuarantees family=A region=77154c9b50d6b1163c1785212a1cb410] 2024-12-04T19:15:25,257 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/A/461d009f76954422b02d34e1df1615ea is 175, key is test_row_0/A:col10/1733339725197/Put/seqid=0 2024-12-04T19:15:25,263 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733339725249, max timestamp=1733339725260Num uniques 859; 2024-12-04T19:15:25,263 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T19:15:25,263 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733339725249, max timestamp=1733339725260Num uniques 859; 2024-12-04T19:15:25,264 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T19:15:25,265 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733339725249, max timestamp=1733339725260Num uniques 859; 2024-12-04T19:15:25,265 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T19:15:25,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742167_1343 (size=30955) 2024-12-04T19:15:25,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-04T19:15:25,317 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:25,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49118 deadline: 1733339785308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:25,318 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:25,318 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:25,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49042 deadline: 1733339785309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:25,318 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:25,318 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:25,318 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:25,318 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:25,318 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:25,320 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:25,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49102 deadline: 1733339785309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:25,321 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:25,321 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:25,321 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:25,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:25,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49134 deadline: 1733339785310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:25,321 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:25,321 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:25,321 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:25,325 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:25,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49070 deadline: 1733339785318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:25,325 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:25,325 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:25,326 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:25,344 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-04T19:15:25,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:25,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. as already flushing 2024-12-04T19:15:25,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:25,344 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:25,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:25,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:25,425 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:25,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49118 deadline: 1733339785422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:25,426 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:25,426 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:25,426 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:25,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49042 deadline: 1733339785422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:25,426 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:25,426 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:25,426 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:25,426 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:25,434 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:25,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49102 deadline: 1733339785431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:25,434 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:25,434 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:25,434 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:25,434 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:25,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49134 deadline: 1733339785431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:25,434 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:25,434 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:25,434 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:25,434 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:25,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49070 deadline: 1733339785431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:25,435 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:25,435 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:25,435 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:25,497 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-04T19:15:25,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:25,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. as already flushing 2024-12-04T19:15:25,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:25,497 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:25,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:25,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:25,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-04T19:15:25,637 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:25,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49118 deadline: 1733339785631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:25,638 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:25,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:25,638 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:25,638 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:25,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49042 deadline: 1733339785631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:25,638 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:25,638 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:25,638 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:25,645 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:25,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49102 deadline: 1733339785641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:25,645 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:25,645 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:25,645 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:25,646 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:25,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49134 deadline: 1733339785641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:25,646 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:25,646 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:25,646 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:25,649 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-04T19:15:25,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:25,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. as already flushing 2024-12-04T19:15:25,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:25,650 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:25,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:25,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:25,652 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:25,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49070 deadline: 1733339785641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:25,652 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:25,652 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:25,652 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:25,675 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/A/461d009f76954422b02d34e1df1615ea 2024-12-04T19:15:25,698 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/B/32ba7b3fa95d42cc9206ac7c1c76ffea is 50, key is test_row_0/B:col10/1733339725197/Put/seqid=0 2024-12-04T19:15:25,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742168_1344 (size=12001) 2024-12-04T19:15:25,802 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-04T19:15:25,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:25,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. as already flushing 2024-12-04T19:15:25,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:25,802 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:25,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:25,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:25,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-04T19:15:25,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:25,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49118 deadline: 1733339785942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:25,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:25,945 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:25,945 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:25,945 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:25,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49042 deadline: 1733339785942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:25,945 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:25,945 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:25,945 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:25,955 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-04T19:15:25,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:25,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. as already flushing 2024-12-04T19:15:25,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:25,955 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:25,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:25,955 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:25,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49102 deadline: 1733339785952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:25,955 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:25,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49134 deadline: 1733339785952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:25,955 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:25,956 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:25,956 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:25,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:25,956 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:25,956 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:25,956 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:25,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:25,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49070 deadline: 1733339785961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:25,964 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:25,964 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:25,964 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:26,101 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/B/32ba7b3fa95d42cc9206ac7c1c76ffea 2024-12-04T19:15:26,106 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-04T19:15:26,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:26,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. as already flushing 2024-12-04T19:15:26,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:26,107 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:26,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:26,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:26,124 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/C/6fa081a0b96449c9adf48594b89e1e5a is 50, key is test_row_0/C:col10/1733339725197/Put/seqid=0 2024-12-04T19:15:26,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742169_1345 (size=12001) 2024-12-04T19:15:26,258 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-04T19:15:26,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:26,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. as already flushing 2024-12-04T19:15:26,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:26,259 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:26,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:26,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:26,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-04T19:15:26,411 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-04T19:15:26,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:26,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. as already flushing 2024-12-04T19:15:26,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:26,412 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:26,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:26,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:26,455 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:26,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49118 deadline: 1733339786452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:26,455 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:26,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49042 deadline: 1733339786452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:26,455 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:26,455 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:26,455 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:26,456 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:26,456 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:26,456 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:26,463 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:26,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49102 deadline: 1733339786461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:26,463 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:26,463 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:26,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49134 deadline: 1733339786461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:26,463 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:26,463 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:26,464 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:26,464 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:26,464 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:26,477 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:26,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49070 deadline: 1733339786472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:26,477 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:26,477 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:26,477 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:26,529 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/C/6fa081a0b96449c9adf48594b89e1e5a 2024-12-04T19:15:26,533 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/A/461d009f76954422b02d34e1df1615ea as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/461d009f76954422b02d34e1df1615ea 2024-12-04T19:15:26,536 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/461d009f76954422b02d34e1df1615ea, entries=150, sequenceid=15, filesize=30.2 K 2024-12-04T19:15:26,537 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/B/32ba7b3fa95d42cc9206ac7c1c76ffea as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/32ba7b3fa95d42cc9206ac7c1c76ffea 2024-12-04T19:15:26,539 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/32ba7b3fa95d42cc9206ac7c1c76ffea, entries=150, sequenceid=15, filesize=11.7 K 2024-12-04T19:15:26,540 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/C/6fa081a0b96449c9adf48594b89e1e5a as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/6fa081a0b96449c9adf48594b89e1e5a 2024-12-04T19:15:26,543 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/6fa081a0b96449c9adf48594b89e1e5a, entries=150, sequenceid=15, filesize=11.7 K 2024-12-04T19:15:26,544 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 77154c9b50d6b1163c1785212a1cb410 in 1343ms, sequenceid=15, compaction requested=false 2024-12-04T19:15:26,544 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-04T19:15:26,544 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 77154c9b50d6b1163c1785212a1cb410: 2024-12-04T19:15:26,564 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-04T19:15:26,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:26,564 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2902): Flushing 77154c9b50d6b1163c1785212a1cb410 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-12-04T19:15:26,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77154c9b50d6b1163c1785212a1cb410, store=A 2024-12-04T19:15:26,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T19:15:26,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77154c9b50d6b1163c1785212a1cb410, store=B 2024-12-04T19:15:26,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T19:15:26,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77154c9b50d6b1163c1785212a1cb410, store=C 2024-12-04T19:15:26,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T19:15:26,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120427eb51b7d2d94a5198d31163db18f57c_77154c9b50d6b1163c1785212a1cb410 is 50, key is test_row_0/A:col10/1733339725308/Put/seqid=0 2024-12-04T19:15:26,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742170_1346 (size=12154) 2024-12-04T19:15:26,967 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-04T19:15:26,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:26,978 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120427eb51b7d2d94a5198d31163db18f57c_77154c9b50d6b1163c1785212a1cb410 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120427eb51b7d2d94a5198d31163db18f57c_77154c9b50d6b1163c1785212a1cb410 2024-12-04T19:15:26,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/A/298ef5210f394fd6a0a38b1d86f97459, store: [table=TestAcidGuarantees family=A region=77154c9b50d6b1163c1785212a1cb410] 2024-12-04T19:15:26,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/A/298ef5210f394fd6a0a38b1d86f97459 is 175, key is test_row_0/A:col10/1733339725308/Put/seqid=0 2024-12-04T19:15:26,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742171_1347 (size=30955) 2024-12-04T19:15:26,991 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=50, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/A/298ef5210f394fd6a0a38b1d86f97459 2024-12-04T19:15:27,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/B/a3424de3a58d438d8828ae52e9bae33a is 50, key is test_row_0/B:col10/1733339725308/Put/seqid=0 2024-12-04T19:15:27,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742172_1348 (size=12001) 2024-12-04T19:15:27,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-04T19:15:27,406 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/B/a3424de3a58d438d8828ae52e9bae33a 2024-12-04T19:15:27,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/C/b07c0982f557424bafb470321f9902b7 is 50, key is test_row_0/C:col10/1733339725308/Put/seqid=0 2024-12-04T19:15:27,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742173_1349 (size=12001) 2024-12-04T19:15:27,417 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/C/b07c0982f557424bafb470321f9902b7 2024-12-04T19:15:27,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/A/298ef5210f394fd6a0a38b1d86f97459 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/298ef5210f394fd6a0a38b1d86f97459 2024-12-04T19:15:27,425 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/298ef5210f394fd6a0a38b1d86f97459, entries=150, sequenceid=50, filesize=30.2 K 2024-12-04T19:15:27,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/B/a3424de3a58d438d8828ae52e9bae33a as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/a3424de3a58d438d8828ae52e9bae33a 2024-12-04T19:15:27,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,429 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/a3424de3a58d438d8828ae52e9bae33a, entries=150, sequenceid=50, filesize=11.7 K 2024-12-04T19:15:27,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/C/b07c0982f557424bafb470321f9902b7 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/b07c0982f557424bafb470321f9902b7 2024-12-04T19:15:27,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,433 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/b07c0982f557424bafb470321f9902b7, entries=150, sequenceid=50, filesize=11.7 K 2024-12-04T19:15:27,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,439 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=0 B/0 for 77154c9b50d6b1163c1785212a1cb410 in 874ms, sequenceid=50, compaction requested=false 2024-12-04T19:15:27,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2603): Flush status journal for 77154c9b50d6b1163c1785212a1cb410: 2024-12-04T19:15:27,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:27,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-12-04T19:15:27,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.HMaster(4169): Remote procedure done, pid=96 2024-12-04T19:15:27,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,442 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-12-04T19:15:27,442 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2480 sec 2024-12-04T19:15:27,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,443 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees in 2.2560 sec 2024-12-04T19:15:27,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(8855): Flush requested on 77154c9b50d6b1163c1785212a1cb410 2024-12-04T19:15:27,534 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 77154c9b50d6b1163c1785212a1cb410 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-04T19:15:27,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77154c9b50d6b1163c1785212a1cb410, store=A 2024-12-04T19:15:27,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:15:27,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77154c9b50d6b1163c1785212a1cb410, store=B 2024-12-04T19:15:27,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:15:27,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77154c9b50d6b1163c1785212a1cb410, store=C 2024-12-04T19:15:27,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:15:27,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,540 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412040e8ecdf10e794d948498c8c6ab0e5e57_77154c9b50d6b1163c1785212a1cb410 is 50, key is test_row_0/A:col10/1733339727529/Put/seqid=0 2024-12-04T19:15:27,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742174_1350 (size=21918) 2024-12-04T19:15:27,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,581 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=859, cellSize=39335, totalHeapSize=89576, min timestamp=1733339727534, max timestamp=1733339727579Num uniques 859; 2024-12-04T19:15:27,581 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=859, cellSize=39335, totalHeapSize=89576, min timestamp=1733339727534, max timestamp=1733339727579Num uniques 859; 2024-12-04T19:15:27,581 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=39335, suffix heap size=117064,new segment heap size=89576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=859 2024-12-04T19:15:27,581 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=39335, suffix heap size=117064,new segment heap size=89576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=859 2024-12-04T19:15:27,581 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=859, cellSize=39335, totalHeapSize=89576, min timestamp=1733339727534, max timestamp=1733339727579Num uniques 859; 2024-12-04T19:15:27,581 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=39335, suffix heap size=117064,new segment heap size=89576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=859 2024-12-04T19:15:27,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,598 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:27,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49118 deadline: 1733339787582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:27,599 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:27,599 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:27,599 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:27,600 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:27,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49042 deadline: 1733339787590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:27,601 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:27,601 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:27,601 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:27,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:27,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49102 deadline: 1733339787591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:27,606 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:27,606 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:27,606 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:27,606 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:27,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49134 deadline: 1733339787593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:27,606 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:27,606 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:27,606 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:27,607 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:27,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49070 deadline: 1733339787599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:27,607 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:27,607 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:27,607 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:27,706 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:27,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49118 deadline: 1733339787701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:27,706 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:27,706 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:27,706 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:27,717 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:27,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49042 deadline: 1733339787711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:27,717 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:27,717 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:27,717 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:27,717 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:27,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49102 deadline: 1733339787711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:27,718 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:27,718 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:27,718 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:27,718 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:27,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49134 deadline: 1733339787712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:27,718 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:27,718 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:27,718 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:27,718 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:27,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49070 deadline: 1733339787712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:27,718 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:27,718 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:27,718 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:27,918 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:27,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49118 deadline: 1733339787911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:27,918 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:27,918 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:27,918 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:27,927 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:27,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49042 deadline: 1733339787921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:27,927 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:27,927 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:27,928 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:27,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49102 deadline: 1733339787921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:27,928 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:27,928 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:27,928 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:27,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:27,928 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:27,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49134 deadline: 1733339787922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:27,928 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:27,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:27,928 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:27,928 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:27,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49070 deadline: 1733339787922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:27,928 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:27,928 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:27,928 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:27,945 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:27,948 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412040e8ecdf10e794d948498c8c6ab0e5e57_77154c9b50d6b1163c1785212a1cb410 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412040e8ecdf10e794d948498c8c6ab0e5e57_77154c9b50d6b1163c1785212a1cb410 2024-12-04T19:15:27,949 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/A/0048142ce36c482da6d2550c83845be6, store: [table=TestAcidGuarantees family=A region=77154c9b50d6b1163c1785212a1cb410] 2024-12-04T19:15:27,949 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/A/0048142ce36c482da6d2550c83845be6 is 175, key is test_row_0/A:col10/1733339727529/Put/seqid=0 2024-12-04T19:15:27,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742175_1351 (size=65323) 2024-12-04T19:15:28,225 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:28,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49118 deadline: 1733339788222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:28,225 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:28,225 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:28,225 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:28,235 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:28,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49042 deadline: 1733339788231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:28,235 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:28,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49102 deadline: 1733339788231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:28,235 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:28,235 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:28,235 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:28,236 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:28,235 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:28,236 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:28,236 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:28,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49070 deadline: 1733339788231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:28,236 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:28,236 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:28,236 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:28,245 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:28,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49134 deadline: 1733339788242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:28,246 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:28,246 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:28,246 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:28,353 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=60, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/A/0048142ce36c482da6d2550c83845be6 2024-12-04T19:15:28,360 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/B/6540f00e49b840bda8e16478daac2a28 is 50, key is test_row_0/B:col10/1733339727529/Put/seqid=0 2024-12-04T19:15:28,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742176_1352 (size=12001) 2024-12-04T19:15:28,737 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:28,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49118 deadline: 1733339788731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:28,737 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:28,737 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:28,737 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:28,745 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:28,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49042 deadline: 1733339788742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:28,745 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:28,745 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:28,745 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:28,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49102 deadline: 1733339788742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:28,745 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:28,746 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:28,746 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:28,746 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:28,746 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:28,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49070 deadline: 1733339788742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:28,746 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:28,746 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:28,746 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:28,757 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:28,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49134 deadline: 1733339788752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:28,757 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:28,757 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:28,757 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:28,763 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/B/6540f00e49b840bda8e16478daac2a28 2024-12-04T19:15:28,768 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/C/72e48841d1804b4c8c543ff781208102 is 50, key is test_row_0/C:col10/1733339727529/Put/seqid=0 2024-12-04T19:15:28,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742177_1353 (size=12001) 2024-12-04T19:15:29,171 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/C/72e48841d1804b4c8c543ff781208102 2024-12-04T19:15:29,232 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/A/0048142ce36c482da6d2550c83845be6 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/0048142ce36c482da6d2550c83845be6 2024-12-04T19:15:29,236 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/0048142ce36c482da6d2550c83845be6, entries=350, sequenceid=60, filesize=63.8 K 2024-12-04T19:15:29,236 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/B/6540f00e49b840bda8e16478daac2a28 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/6540f00e49b840bda8e16478daac2a28 2024-12-04T19:15:29,240 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/6540f00e49b840bda8e16478daac2a28, entries=150, sequenceid=60, filesize=11.7 K 2024-12-04T19:15:29,241 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/C/72e48841d1804b4c8c543ff781208102 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/72e48841d1804b4c8c543ff781208102 2024-12-04T19:15:29,244 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/72e48841d1804b4c8c543ff781208102, entries=150, sequenceid=60, filesize=11.7 K 2024-12-04T19:15:29,244 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=154.31 KB/158010 for 77154c9b50d6b1163c1785212a1cb410 in 1710ms, sequenceid=60, compaction requested=true 2024-12-04T19:15:29,244 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 77154c9b50d6b1163c1785212a1cb410: 2024-12-04T19:15:29,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 77154c9b50d6b1163c1785212a1cb410:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T19:15:29,245 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:15:29,245 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 77154c9b50d6b1163c1785212a1cb410:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T19:15:29,245 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T19:15:29,245 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:15:29,245 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T19:15:29,245 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 77154c9b50d6b1163c1785212a1cb410:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T19:15:29,245 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T19:15:29,245 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T19:15:29,245 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 127233 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T19:15:29,245 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1541): 77154c9b50d6b1163c1785212a1cb410/B is initiating minor compaction (all files) 2024-12-04T19:15:29,245 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1541): 77154c9b50d6b1163c1785212a1cb410/A is initiating minor compaction (all files) 2024-12-04T19:15:29,245 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 77154c9b50d6b1163c1785212a1cb410/B in TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:29,245 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 77154c9b50d6b1163c1785212a1cb410/A in TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:29,245 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/32ba7b3fa95d42cc9206ac7c1c76ffea, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/a3424de3a58d438d8828ae52e9bae33a, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/6540f00e49b840bda8e16478daac2a28] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp, totalSize=35.2 K 2024-12-04T19:15:29,246 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/461d009f76954422b02d34e1df1615ea, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/298ef5210f394fd6a0a38b1d86f97459, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/0048142ce36c482da6d2550c83845be6] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp, totalSize=124.3 K 2024-12-04T19:15:29,246 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:29,246 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. files: [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/461d009f76954422b02d34e1df1615ea, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/298ef5210f394fd6a0a38b1d86f97459, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/0048142ce36c482da6d2550c83845be6] 2024-12-04T19:15:29,246 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 32ba7b3fa95d42cc9206ac7c1c76ffea, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733339725194 2024-12-04T19:15:29,246 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 461d009f76954422b02d34e1df1615ea, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733339725194 2024-12-04T19:15:29,246 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting a3424de3a58d438d8828ae52e9bae33a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733339725307 2024-12-04T19:15:29,246 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 6540f00e49b840bda8e16478daac2a28, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733339727518 2024-12-04T19:15:29,246 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 298ef5210f394fd6a0a38b1d86f97459, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733339725307 2024-12-04T19:15:29,247 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0048142ce36c482da6d2550c83845be6, keycount=350, bloomtype=ROW, size=63.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733339727518 2024-12-04T19:15:29,259 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 77154c9b50d6b1163c1785212a1cb410#B#compaction#302 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:15:29,259 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/B/920a4f50be99442191f5c925fec89136 is 50, key is test_row_0/B:col10/1733339727529/Put/seqid=0 2024-12-04T19:15:29,260 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=77154c9b50d6b1163c1785212a1cb410] 2024-12-04T19:15:29,264 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412044640d84e6cb64d2aa28c2865814f10aa_77154c9b50d6b1163c1785212a1cb410 store=[table=TestAcidGuarantees family=A region=77154c9b50d6b1163c1785212a1cb410] 2024-12-04T19:15:29,266 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412044640d84e6cb64d2aa28c2865814f10aa_77154c9b50d6b1163c1785212a1cb410, store=[table=TestAcidGuarantees family=A region=77154c9b50d6b1163c1785212a1cb410] 2024-12-04T19:15:29,266 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412044640d84e6cb64d2aa28c2865814f10aa_77154c9b50d6b1163c1785212a1cb410 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=77154c9b50d6b1163c1785212a1cb410] 2024-12-04T19:15:29,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742178_1354 (size=4469) 2024-12-04T19:15:29,312 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 77154c9b50d6b1163c1785212a1cb410#A#compaction#303 average throughput is 0.47 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:15:29,313 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/A/1e335ddc0d804bb6bd12adcc4a85f1e7 is 175, key is test_row_0/A:col10/1733339727529/Put/seqid=0 2024-12-04T19:15:29,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742179_1355 (size=12104) 2024-12-04T19:15:29,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742180_1356 (size=31058) 2024-12-04T19:15:29,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-04T19:15:29,352 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T19:15:29,352 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T19:15:29,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-12-04T19:15:29,354 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T19:15:29,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-04T19:15:29,354 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T19:15:29,354 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T19:15:29,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-04T19:15:29,506 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-04T19:15:29,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:29,507 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing 77154c9b50d6b1163c1785212a1cb410 3/3 column families, dataSize=154.31 KB heapSize=379.13 KB 2024-12-04T19:15:29,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77154c9b50d6b1163c1785212a1cb410, store=A 2024-12-04T19:15:29,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T19:15:29,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77154c9b50d6b1163c1785212a1cb410, store=B 2024-12-04T19:15:29,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T19:15:29,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77154c9b50d6b1163c1785212a1cb410, store=C 2024-12-04T19:15:29,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T19:15:29,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412043f9fd9b27f9e4049ad6e219d5a137ef8_77154c9b50d6b1163c1785212a1cb410 is 50, key is test_row_0/A:col10/1733339727591/Put/seqid=0 2024-12-04T19:15:29,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742181_1357 (size=12154) 2024-12-04T19:15:29,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-04T19:15:29,718 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/B/920a4f50be99442191f5c925fec89136 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/920a4f50be99442191f5c925fec89136 2024-12-04T19:15:29,722 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 77154c9b50d6b1163c1785212a1cb410/B of 77154c9b50d6b1163c1785212a1cb410 into 920a4f50be99442191f5c925fec89136(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:15:29,722 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 77154c9b50d6b1163c1785212a1cb410: 2024-12-04T19:15:29,722 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., storeName=77154c9b50d6b1163c1785212a1cb410/B, priority=13, startTime=1733339729245; duration=0sec 2024-12-04T19:15:29,722 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T19:15:29,722 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 77154c9b50d6b1163c1785212a1cb410:B 2024-12-04T19:15:29,722 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T19:15:29,723 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T19:15:29,723 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1541): 77154c9b50d6b1163c1785212a1cb410/C is initiating minor compaction (all files) 2024-12-04T19:15:29,723 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 77154c9b50d6b1163c1785212a1cb410/C in TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:29,723 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/6fa081a0b96449c9adf48594b89e1e5a, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/b07c0982f557424bafb470321f9902b7, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/72e48841d1804b4c8c543ff781208102] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp, totalSize=35.2 K 2024-12-04T19:15:29,724 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 6fa081a0b96449c9adf48594b89e1e5a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733339725194 2024-12-04T19:15:29,724 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting b07c0982f557424bafb470321f9902b7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733339725307 2024-12-04T19:15:29,724 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 72e48841d1804b4c8c543ff781208102, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733339727518 2024-12-04T19:15:29,726 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/A/1e335ddc0d804bb6bd12adcc4a85f1e7 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/1e335ddc0d804bb6bd12adcc4a85f1e7 2024-12-04T19:15:29,731 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 77154c9b50d6b1163c1785212a1cb410#C#compaction#305 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:15:29,731 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 77154c9b50d6b1163c1785212a1cb410/A of 77154c9b50d6b1163c1785212a1cb410 into 1e335ddc0d804bb6bd12adcc4a85f1e7(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:15:29,731 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 77154c9b50d6b1163c1785212a1cb410: 2024-12-04T19:15:29,731 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., storeName=77154c9b50d6b1163c1785212a1cb410/A, priority=13, startTime=1733339729244; duration=0sec 2024-12-04T19:15:29,731 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:15:29,731 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 77154c9b50d6b1163c1785212a1cb410:A 2024-12-04T19:15:29,731 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/C/891d348ee4974c1ebf10503be913cd89 is 50, key is test_row_0/C:col10/1733339727529/Put/seqid=0 2024-12-04T19:15:29,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742182_1358 (size=12104) 2024-12-04T19:15:29,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(8855): Flush requested on 77154c9b50d6b1163c1785212a1cb410 2024-12-04T19:15:29,755 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. as already flushing 2024-12-04T19:15:29,766 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:29,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49042 deadline: 1733339789762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:29,767 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:29,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49134 deadline: 1733339789762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:29,767 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:29,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49102 deadline: 1733339789764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:29,767 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 9ba2734cd39f,33969,1733339629791 for 'test_row_2' in 77154c9b50d6b1163c1785212a1cb410 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-04T19:15:29,767 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:29,767 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:29,767 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:29,767 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 9ba2734cd39f,33969,1733339629791 for 'test_row_1' in 77154c9b50d6b1163c1785212a1cb410 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2174 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-04T19:15:29,767 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:29,767 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:29,767 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:29,767 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:29,767 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:29,767 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:29,772 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:29,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49070 deadline: 1733339789765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:29,772 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:29,772 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:29,772 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:29,775 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:29,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49118 deadline: 1733339789767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:29,775 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:29,775 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:29,775 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:29,877 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:29,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49102 deadline: 1733339789871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:29,877 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:29,877 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:29,877 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:29,885 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:29,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49070 deadline: 1733339789882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:29,886 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:29,886 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:29,886 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:29,886 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:29,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49118 deadline: 1733339789882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:29,886 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:29,886 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:29,886 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:29,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:29,919 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412043f9fd9b27f9e4049ad6e219d5a137ef8_77154c9b50d6b1163c1785212a1cb410 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412043f9fd9b27f9e4049ad6e219d5a137ef8_77154c9b50d6b1163c1785212a1cb410 2024-12-04T19:15:29,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/A/bd06c8d06ec74bd5a779b9035c601b68, store: [table=TestAcidGuarantees family=A region=77154c9b50d6b1163c1785212a1cb410] 2024-12-04T19:15:29,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/A/bd06c8d06ec74bd5a779b9035c601b68 is 175, key is test_row_0/A:col10/1733339727591/Put/seqid=0 2024-12-04T19:15:29,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742183_1359 (size=30955) 2024-12-04T19:15:29,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-04T19:15:30,084 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:30,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49102 deadline: 1733339790082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:30,084 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:30,084 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:30,084 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:30,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:30,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49070 deadline: 1733339790092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:30,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:30,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:30,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:30,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49118 deadline: 1733339790092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:30,099 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:30,099 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:30,099 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:30,099 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:30,139 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/C/891d348ee4974c1ebf10503be913cd89 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/891d348ee4974c1ebf10503be913cd89 2024-12-04T19:15:30,143 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 77154c9b50d6b1163c1785212a1cb410/C of 77154c9b50d6b1163c1785212a1cb410 into 891d348ee4974c1ebf10503be913cd89(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:15:30,143 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 77154c9b50d6b1163c1785212a1cb410: 2024-12-04T19:15:30,143 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., storeName=77154c9b50d6b1163c1785212a1cb410/C, priority=13, startTime=1733339729245; duration=0sec 2024-12-04T19:15:30,143 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:15:30,143 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 77154c9b50d6b1163c1785212a1cb410:C 2024-12-04T19:15:30,324 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=86, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/A/bd06c8d06ec74bd5a779b9035c601b68 2024-12-04T19:15:30,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/B/091471b12ef44197b69e5d1e592c805d is 50, key is test_row_0/B:col10/1733339727591/Put/seqid=0 2024-12-04T19:15:30,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742184_1360 (size=12001) 2024-12-04T19:15:30,397 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:30,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49102 deadline: 1733339790392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:30,397 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:30,397 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:30,397 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:30,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:30,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49070 deadline: 1733339790401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:30,405 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:30,405 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:30,405 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:30,407 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:30,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49118 deadline: 1733339790401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:30,407 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:30,407 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:30,407 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:30,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-04T19:15:30,734 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/B/091471b12ef44197b69e5d1e592c805d 2024-12-04T19:15:30,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/C/6db98868848b43cd9be3384962ea0010 is 50, key is test_row_0/C:col10/1733339727591/Put/seqid=0 2024-12-04T19:15:30,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742185_1361 (size=12001) 2024-12-04T19:15:30,905 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:30,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49102 deadline: 1733339790901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:30,906 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:30,906 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:30,906 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:30,918 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:30,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49070 deadline: 1733339790911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:30,918 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:30,918 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:30,918 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:30,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49118 deadline: 1733339790911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:30,919 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:30,919 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:30,919 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:30,919 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:31,145 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/C/6db98868848b43cd9be3384962ea0010 2024-12-04T19:15:31,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/A/bd06c8d06ec74bd5a779b9035c601b68 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/bd06c8d06ec74bd5a779b9035c601b68 2024-12-04T19:15:31,153 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/bd06c8d06ec74bd5a779b9035c601b68, entries=150, sequenceid=86, filesize=30.2 K 2024-12-04T19:15:31,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/B/091471b12ef44197b69e5d1e592c805d as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/091471b12ef44197b69e5d1e592c805d 2024-12-04T19:15:31,159 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/091471b12ef44197b69e5d1e592c805d, entries=150, sequenceid=86, filesize=11.7 K 2024-12-04T19:15:31,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/C/6db98868848b43cd9be3384962ea0010 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/6db98868848b43cd9be3384962ea0010 2024-12-04T19:15:31,162 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/6db98868848b43cd9be3384962ea0010, entries=150, sequenceid=86, filesize=11.7 K 2024-12-04T19:15:31,163 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~379.08 KB/388176, currentSize=46.96 KB/48090 for 77154c9b50d6b1163c1785212a1cb410 in 1657ms, sequenceid=86, compaction requested=false 2024-12-04T19:15:31,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for 77154c9b50d6b1163c1785212a1cb410: 2024-12-04T19:15:31,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:31,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-12-04T19:15:31,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-12-04T19:15:31,166 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-12-04T19:15:31,166 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8100 sec 2024-12-04T19:15:31,168 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 1.8150 sec 2024-12-04T19:15:31,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-04T19:15:31,502 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T19:15:31,502 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T19:15:31,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-12-04T19:15:31,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-04T19:15:31,504 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T19:15:31,504 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T19:15:31,504 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T19:15:31,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-04T19:15:31,655 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-04T19:15:31,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:31,656 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing 77154c9b50d6b1163c1785212a1cb410 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-04T19:15:31,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77154c9b50d6b1163c1785212a1cb410, store=A 2024-12-04T19:15:31,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:15:31,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77154c9b50d6b1163c1785212a1cb410, store=B 2024-12-04T19:15:31,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:15:31,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77154c9b50d6b1163c1785212a1cb410, store=C 2024-12-04T19:15:31,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:15:31,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120425cff7db38a440999ae77139441fdb88_77154c9b50d6b1163c1785212a1cb410 is 50, key is test_row_0/A:col10/1733339729756/Put/seqid=0 2024-12-04T19:15:31,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742186_1362 (size=12154) 2024-12-04T19:15:31,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:31,667 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120425cff7db38a440999ae77139441fdb88_77154c9b50d6b1163c1785212a1cb410 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120425cff7db38a440999ae77139441fdb88_77154c9b50d6b1163c1785212a1cb410 2024-12-04T19:15:31,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/A/fc62085152a9490baf45161760aaad8f, store: [table=TestAcidGuarantees family=A region=77154c9b50d6b1163c1785212a1cb410] 2024-12-04T19:15:31,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/A/fc62085152a9490baf45161760aaad8f is 175, key is test_row_0/A:col10/1733339729756/Put/seqid=0 2024-12-04T19:15:31,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742187_1363 (size=30955) 2024-12-04T19:15:31,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(8855): Flush requested on 77154c9b50d6b1163c1785212a1cb410 2024-12-04T19:15:31,786 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. as already flushing 2024-12-04T19:15:31,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-04T19:15:31,843 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733339731823, max timestamp=1733339731840Num uniques 859; 2024-12-04T19:15:31,843 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733339731823, max timestamp=1733339731840Num uniques 859; 2024-12-04T19:15:31,843 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733339731823, max timestamp=1733339731840Num uniques 859; 2024-12-04T19:15:31,843 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T19:15:31,843 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T19:15:31,843 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T19:15:31,913 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:31,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49042 deadline: 1733339791903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:31,914 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:31,914 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:31,914 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:31,919 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:31,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49134 deadline: 1733339791911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:31,920 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:31,920 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:31,920 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:31,929 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:31,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49102 deadline: 1733339791922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:31,929 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:31,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49070 deadline: 1733339791922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:31,929 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 9ba2734cd39f,33969,1733339629791 for 'test_row_2' in 77154c9b50d6b1163c1785212a1cb410 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-04T19:15:31,930 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 9ba2734cd39f,33969,1733339629791 for 'test_row_0' in 77154c9b50d6b1163c1785212a1cb410 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-04T19:15:31,930 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:31,930 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:31,930 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:31,930 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:31,930 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:31,930 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:31,938 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:31,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49118 deadline: 1733339791931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:31,939 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 9ba2734cd39f,33969,1733339629791 for 'test_row_1' in 77154c9b50d6b1163c1785212a1cb410 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2172 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-04T19:15:31,939 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:31,939 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:31,939 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:32,028 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:32,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49042 deadline: 1733339792021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:32,028 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:32,028 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:32,028 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:32,028 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:32,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49134 deadline: 1733339792021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:32,028 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:32,029 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:32,029 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:32,071 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=99, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/A/fc62085152a9490baf45161760aaad8f 2024-12-04T19:15:32,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/B/95edfde51af24d9187456293eff41895 is 50, key is test_row_0/B:col10/1733339729756/Put/seqid=0 2024-12-04T19:15:32,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742188_1364 (size=12001) 2024-12-04T19:15:32,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-04T19:15:32,239 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:32,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49042 deadline: 1733339792232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:32,240 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:32,240 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:32,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49134 deadline: 1733339792232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:32,240 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:32,240 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:32,240 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:32,240 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:32,240 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:32,480 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/B/95edfde51af24d9187456293eff41895 2024-12-04T19:15:32,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/C/29d5215236f54d418ed2a5a5985a9903 is 50, key is test_row_0/C:col10/1733339729756/Put/seqid=0 2024-12-04T19:15:32,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742189_1365 (size=12001) 2024-12-04T19:15:32,560 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:32,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49042 deadline: 1733339792551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:32,561 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:32,561 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:32,561 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:32,561 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:32,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49134 deadline: 1733339792551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:32,561 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:32,561 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:32,561 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:32,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-04T19:15:32,889 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/C/29d5215236f54d418ed2a5a5985a9903 2024-12-04T19:15:32,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/A/fc62085152a9490baf45161760aaad8f as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/fc62085152a9490baf45161760aaad8f 2024-12-04T19:15:32,895 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/fc62085152a9490baf45161760aaad8f, entries=150, sequenceid=99, filesize=30.2 K 2024-12-04T19:15:32,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/B/95edfde51af24d9187456293eff41895 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/95edfde51af24d9187456293eff41895 2024-12-04T19:15:32,900 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/95edfde51af24d9187456293eff41895, entries=150, sequenceid=99, filesize=11.7 K 2024-12-04T19:15:32,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/C/29d5215236f54d418ed2a5a5985a9903 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/29d5215236f54d418ed2a5a5985a9903 2024-12-04T19:15:32,904 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/29d5215236f54d418ed2a5a5985a9903, entries=150, sequenceid=99, filesize=11.7 K 2024-12-04T19:15:32,905 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 77154c9b50d6b1163c1785212a1cb410 in 1250ms, sequenceid=99, compaction requested=true 2024-12-04T19:15:32,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for 77154c9b50d6b1163c1785212a1cb410: 2024-12-04T19:15:32,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:32,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-12-04T19:15:32,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-12-04T19:15:32,908 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-12-04T19:15:32,908 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4020 sec 2024-12-04T19:15:32,910 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 1.4070 sec 2024-12-04T19:15:33,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(8855): Flush requested on 77154c9b50d6b1163c1785212a1cb410 2024-12-04T19:15:33,079 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 77154c9b50d6b1163c1785212a1cb410 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-12-04T19:15:33,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77154c9b50d6b1163c1785212a1cb410, store=A 2024-12-04T19:15:33,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T19:15:33,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77154c9b50d6b1163c1785212a1cb410, store=B 2024-12-04T19:15:33,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T19:15:33,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77154c9b50d6b1163c1785212a1cb410, store=C 2024-12-04T19:15:33,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T19:15:33,086 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412048854e24879fa4e9bb80ed65ebca695a5_77154c9b50d6b1163c1785212a1cb410 is 50, key is test_row_0/A:col10/1733339733078/Put/seqid=0 2024-12-04T19:15:33,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742190_1366 (size=14794) 2024-12-04T19:15:33,092 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,095 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412048854e24879fa4e9bb80ed65ebca695a5_77154c9b50d6b1163c1785212a1cb410 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412048854e24879fa4e9bb80ed65ebca695a5_77154c9b50d6b1163c1785212a1cb410 2024-12-04T19:15:33,095 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/A/36032633cf874735889f0f4a857f573c, store: [table=TestAcidGuarantees family=A region=77154c9b50d6b1163c1785212a1cb410] 2024-12-04T19:15:33,096 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/A/36032633cf874735889f0f4a857f573c is 175, key is test_row_0/A:col10/1733339733078/Put/seqid=0 2024-12-04T19:15:33,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742191_1367 (size=39749) 2024-12-04T19:15:33,099 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=136, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/A/36032633cf874735889f0f4a857f573c 2024-12-04T19:15:33,105 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/B/49d869b5512f435c947da2b682ded9ec is 50, key is test_row_0/B:col10/1733339733078/Put/seqid=0 2024-12-04T19:15:33,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742192_1368 (size=12151) 2024-12-04T19:15:33,112 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:33,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49134 deadline: 1733339793105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:33,113 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:33,113 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:33,113 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:33,126 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:33,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49042 deadline: 1733339793113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:33,127 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:33,127 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:33,127 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:33,228 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:33,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49134 deadline: 1733339793222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:33,229 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:33,229 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:33,229 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:33,239 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:33,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49042 deadline: 1733339793232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:33,239 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:33,239 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:33,239 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:33,435 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:33,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49134 deadline: 1733339793431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:33,435 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:33,435 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:33,436 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:33,447 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:33,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49042 deadline: 1733339793442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:33,448 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:33,448 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:33,448 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:33,508 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/B/49d869b5512f435c947da2b682ded9ec 2024-12-04T19:15:33,515 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/C/b9d34dd534aa40a38da0255b7ff2a80e is 50, key is test_row_0/C:col10/1733339733078/Put/seqid=0 2024-12-04T19:15:33,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742193_1369 (size=12151) 2024-12-04T19:15:33,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-04T19:15:33,651 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T19:15:33,652 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T19:15:33,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-12-04T19:15:33,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-04T19:15:33,653 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T19:15:33,654 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T19:15:33,654 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T19:15:33,747 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:33,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49134 deadline: 1733339793742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:33,747 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:33,747 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:33,747 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:33,757 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:33,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49042 deadline: 1733339793752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:33,758 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:33,758 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:33,758 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:33,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-04T19:15:33,806 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-04T19:15:33,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:33,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. as already flushing 2024-12-04T19:15:33,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:33,806 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:33,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:33,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:33,922 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/C/b9d34dd534aa40a38da0255b7ff2a80e 2024-12-04T19:15:33,926 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/A/36032633cf874735889f0f4a857f573c as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/36032633cf874735889f0f4a857f573c 2024-12-04T19:15:33,928 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/36032633cf874735889f0f4a857f573c, entries=200, sequenceid=136, filesize=38.8 K 2024-12-04T19:15:33,929 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/B/49d869b5512f435c947da2b682ded9ec as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/49d869b5512f435c947da2b682ded9ec 2024-12-04T19:15:33,932 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/49d869b5512f435c947da2b682ded9ec, entries=150, sequenceid=136, filesize=11.9 K 2024-12-04T19:15:33,933 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/C/b9d34dd534aa40a38da0255b7ff2a80e as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/b9d34dd534aa40a38da0255b7ff2a80e 2024-12-04T19:15:33,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,936 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/b9d34dd534aa40a38da0255b7ff2a80e, entries=150, sequenceid=136, filesize=11.9 K 2024-12-04T19:15:33,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,937 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for 77154c9b50d6b1163c1785212a1cb410 in 858ms, sequenceid=136, compaction requested=true 2024-12-04T19:15:33,937 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 77154c9b50d6b1163c1785212a1cb410: 2024-12-04T19:15:33,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 77154c9b50d6b1163c1785212a1cb410:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T19:15:33,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:15:33,937 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T19:15:33,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 77154c9b50d6b1163c1785212a1cb410:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T19:15:33,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:15:33,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 77154c9b50d6b1163c1785212a1cb410:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T19:15:33,937 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T19:15:33,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T19:15:33,938 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 132717 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T19:15:33,938 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48257 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T19:15:33,938 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1541): 77154c9b50d6b1163c1785212a1cb410/A is initiating minor compaction (all files) 2024-12-04T19:15:33,938 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1541): 77154c9b50d6b1163c1785212a1cb410/B is initiating minor compaction (all files) 2024-12-04T19:15:33,938 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 77154c9b50d6b1163c1785212a1cb410/B in TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:33,938 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 77154c9b50d6b1163c1785212a1cb410/A in TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:33,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,938 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/920a4f50be99442191f5c925fec89136, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/091471b12ef44197b69e5d1e592c805d, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/95edfde51af24d9187456293eff41895, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/49d869b5512f435c947da2b682ded9ec] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp, totalSize=47.1 K 2024-12-04T19:15:33,938 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/1e335ddc0d804bb6bd12adcc4a85f1e7, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/bd06c8d06ec74bd5a779b9035c601b68, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/fc62085152a9490baf45161760aaad8f, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/36032633cf874735889f0f4a857f573c] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp, totalSize=129.6 K 2024-12-04T19:15:33,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,939 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:33,939 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. files: [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/1e335ddc0d804bb6bd12adcc4a85f1e7, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/bd06c8d06ec74bd5a779b9035c601b68, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/fc62085152a9490baf45161760aaad8f, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/36032633cf874735889f0f4a857f573c] 2024-12-04T19:15:33,939 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 920a4f50be99442191f5c925fec89136, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733339727518 2024-12-04T19:15:33,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,939 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e335ddc0d804bb6bd12adcc4a85f1e7, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733339727518 2024-12-04T19:15:33,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,939 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 091471b12ef44197b69e5d1e592c805d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1733339727579 2024-12-04T19:15:33,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,939 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting bd06c8d06ec74bd5a779b9035c601b68, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1733339727579 2024-12-04T19:15:33,940 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 95edfde51af24d9187456293eff41895, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1733339729756 2024-12-04T19:15:33,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,940 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting fc62085152a9490baf45161760aaad8f, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1733339729756 2024-12-04T19:15:33,940 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 49d869b5512f435c947da2b682ded9ec, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1733339731888 2024-12-04T19:15:33,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,940 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 36032633cf874735889f0f4a857f573c, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1733339731888 2024-12-04T19:15:33,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,947 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=77154c9b50d6b1163c1785212a1cb410] 2024-12-04T19:15:33,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,948 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 77154c9b50d6b1163c1785212a1cb410#B#compaction#314 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:15:33,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,948 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/B/40b43faf0c7540bb9493721e67ca558b is 50, key is test_row_0/B:col10/1733339733078/Put/seqid=0 2024-12-04T19:15:33,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,953 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241204141f01b8e5624bf7befe2da2927f5c9e_77154c9b50d6b1163c1785212a1cb410 store=[table=TestAcidGuarantees family=A region=77154c9b50d6b1163c1785212a1cb410] 2024-12-04T19:15:33,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,955 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241204141f01b8e5624bf7befe2da2927f5c9e_77154c9b50d6b1163c1785212a1cb410, store=[table=TestAcidGuarantees family=A region=77154c9b50d6b1163c1785212a1cb410] 2024-12-04T19:15:33,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,955 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204141f01b8e5624bf7befe2da2927f5c9e_77154c9b50d6b1163c1785212a1cb410 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=77154c9b50d6b1163c1785212a1cb410] 2024-12-04T19:15:33,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742194_1370 (size=12391) 2024-12-04T19:15:33,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,958 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-04T19:15:33,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:33,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,959 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing 77154c9b50d6b1163c1785212a1cb410 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-12-04T19:15:33,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77154c9b50d6b1163c1785212a1cb410, store=A 2024-12-04T19:15:33,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:15:33,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77154c9b50d6b1163c1785212a1cb410, store=B 2024-12-04T19:15:33,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:15:33,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77154c9b50d6b1163c1785212a1cb410, store=C 2024-12-04T19:15:33,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:15:33,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,962 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/B/40b43faf0c7540bb9493721e67ca558b as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/40b43faf0c7540bb9493721e67ca558b 2024-12-04T19:15:33,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,965 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 77154c9b50d6b1163c1785212a1cb410/B of 77154c9b50d6b1163c1785212a1cb410 into 40b43faf0c7540bb9493721e67ca558b(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:15:33,965 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 77154c9b50d6b1163c1785212a1cb410: 2024-12-04T19:15:33,965 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., storeName=77154c9b50d6b1163c1785212a1cb410/B, priority=12, startTime=1733339733937; duration=0sec 2024-12-04T19:15:33,965 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T19:15:33,966 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 77154c9b50d6b1163c1785212a1cb410:B 2024-12-04T19:15:33,966 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T19:15:33,968 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48257 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T19:15:33,968 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1541): 77154c9b50d6b1163c1785212a1cb410/C is initiating minor compaction (all files) 2024-12-04T19:15:33,968 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 77154c9b50d6b1163c1785212a1cb410/C in TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:33,968 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/891d348ee4974c1ebf10503be913cd89, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/6db98868848b43cd9be3384962ea0010, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/29d5215236f54d418ed2a5a5985a9903, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/b9d34dd534aa40a38da0255b7ff2a80e] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp, totalSize=47.1 K 2024-12-04T19:15:33,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(8855): Flush requested on 77154c9b50d6b1163c1785212a1cb410 2024-12-04T19:15:33,969 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. as already flushing 2024-12-04T19:15:33,968 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 891d348ee4974c1ebf10503be913cd89, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733339727518 2024-12-04T19:15:33,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742195_1371 (size=4469) 2024-12-04T19:15:33,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412045c44cb2ad8d8486885df4e118d9f6e2f_77154c9b50d6b1163c1785212a1cb410 is 50, key is test_row_0/A:col10/1733339733104/Put/seqid=0 2024-12-04T19:15:33,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-04T19:15:33,972 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 6db98868848b43cd9be3384962ea0010, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1733339727579 2024-12-04T19:15:33,972 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 29d5215236f54d418ed2a5a5985a9903, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1733339729756 2024-12-04T19:15:33,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,973 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting b9d34dd534aa40a38da0255b7ff2a80e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1733339731888 2024-12-04T19:15:33,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,982 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 77154c9b50d6b1163c1785212a1cb410#C#compaction#317 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:15:33,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,983 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/C/c7361166861f48bfbba7bbd4f9db6381 is 50, key is test_row_0/C:col10/1733339733078/Put/seqid=0 2024-12-04T19:15:33,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742196_1372 (size=9814) 2024-12-04T19:15:33,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,988 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412045c44cb2ad8d8486885df4e118d9f6e2f_77154c9b50d6b1163c1785212a1cb410 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412045c44cb2ad8d8486885df4e118d9f6e2f_77154c9b50d6b1163c1785212a1cb410 2024-12-04T19:15:33,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/A/ecc20430f3924e45a04862bd2db529cc, store: [table=TestAcidGuarantees family=A region=77154c9b50d6b1163c1785212a1cb410] 2024-12-04T19:15:33,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/A/ecc20430f3924e45a04862bd2db529cc is 175, key is test_row_0/A:col10/1733339733104/Put/seqid=0 2024-12-04T19:15:33,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742197_1373 (size=12391) 2024-12-04T19:15:33,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742198_1374 (size=22461) 2024-12-04T19:15:33,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:33,999 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/C/c7361166861f48bfbba7bbd4f9db6381 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/c7361166861f48bfbba7bbd4f9db6381 2024-12-04T19:15:33,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:34,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:34,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:34,004 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 77154c9b50d6b1163c1785212a1cb410/C of 77154c9b50d6b1163c1785212a1cb410 into c7361166861f48bfbba7bbd4f9db6381(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:15:34,004 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 77154c9b50d6b1163c1785212a1cb410: 2024-12-04T19:15:34,004 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., storeName=77154c9b50d6b1163c1785212a1cb410/C, priority=12, startTime=1733339733937; duration=0sec 2024-12-04T19:15:34,004 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:15:34,004 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 77154c9b50d6b1163c1785212a1cb410:C 2024-12-04T19:15:34,031 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733339734006, max timestamp=1733339734029Num uniques 859; 2024-12-04T19:15:34,031 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-04T19:15:34,032 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733339734006, max timestamp=1733339734029Num uniques 859; 2024-12-04T19:15:34,032 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-04T19:15:34,032 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733339734006, max timestamp=1733339734029Num uniques 859; 2024-12-04T19:15:34,032 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-04T19:15:34,097 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:34,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49070 deadline: 1733339794088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:34,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:34,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:34,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:34,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:34,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49102 deadline: 1733339794089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:34,098 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:34,098 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:34,098 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:34,109 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:34,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49118 deadline: 1733339794097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:34,109 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:34,109 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:34,109 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:34,207 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:34,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49070 deadline: 1733339794201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:34,207 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:34,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49102 deadline: 1733339794201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:34,207 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:34,207 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:34,207 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:34,207 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:34,207 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:34,207 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:34,219 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:34,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49118 deadline: 1733339794212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:34,219 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:34,219 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:34,219 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:34,257 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:34,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49134 deadline: 1733339794252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:34,258 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:34,258 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:34,258 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:34,266 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:34,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49042 deadline: 1733339794261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:34,267 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:34,267 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:34,267 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:34,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-04T19:15:34,373 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 77154c9b50d6b1163c1785212a1cb410#A#compaction#315 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:15:34,373 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/A/1cea96d199fc422ebf161ecf2a2735c7 is 175, key is test_row_0/A:col10/1733339733078/Put/seqid=0 2024-12-04T19:15:34,395 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=145, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/A/ecc20430f3924e45a04862bd2db529cc 2024-12-04T19:15:34,419 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:34,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49070 deadline: 1733339794412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:34,419 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:34,419 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:34,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49102 deadline: 1733339794412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:34,419 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:34,419 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:34,419 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:34,419 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:34,419 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:34,427 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:34,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49118 deadline: 1733339794422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:34,427 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:34,428 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:34,428 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:34,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742199_1375 (size=31345) 2024-12-04T19:15:34,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/B/60166ae1812f4567a6a9f7542e53a72e is 50, key is test_row_0/B:col10/1733339733104/Put/seqid=0 2024-12-04T19:15:34,438 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/A/1cea96d199fc422ebf161ecf2a2735c7 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/1cea96d199fc422ebf161ecf2a2735c7 2024-12-04T19:15:34,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742200_1376 (size=9757) 2024-12-04T19:15:34,451 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 77154c9b50d6b1163c1785212a1cb410/A of 77154c9b50d6b1163c1785212a1cb410 into 1cea96d199fc422ebf161ecf2a2735c7(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:15:34,451 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 77154c9b50d6b1163c1785212a1cb410: 2024-12-04T19:15:34,451 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., storeName=77154c9b50d6b1163c1785212a1cb410/A, priority=12, startTime=1733339733937; duration=0sec 2024-12-04T19:15:34,451 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:15:34,451 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 77154c9b50d6b1163c1785212a1cb410:A 2024-12-04T19:15:34,724 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:34,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49102 deadline: 1733339794722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:34,724 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:34,724 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:34,725 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:34,738 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:34,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49070 deadline: 1733339794732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:34,739 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:34,739 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:34,739 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:34,739 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:34,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49118 deadline: 1733339794732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:34,739 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:34,739 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:34,739 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:34,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-04T19:15:34,845 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/B/60166ae1812f4567a6a9f7542e53a72e 2024-12-04T19:15:34,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/C/74cf0ea3af6d442fb4b0e070a3e01a20 is 50, key is test_row_0/C:col10/1733339733104/Put/seqid=0 2024-12-04T19:15:34,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742201_1377 (size=9757) 2024-12-04T19:15:35,238 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:35,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49102 deadline: 1733339795232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:35,238 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:35,238 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:35,238 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:35,258 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/C/74cf0ea3af6d442fb4b0e070a3e01a20 2024-12-04T19:15:35,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:35,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49070 deadline: 1733339795252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:35,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:35,258 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:35,258 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:35,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49118 deadline: 1733339795252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:35,258 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:35,259 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:35,259 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:35,259 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:35,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/A/ecc20430f3924e45a04862bd2db529cc as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/ecc20430f3924e45a04862bd2db529cc 2024-12-04T19:15:35,268 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/ecc20430f3924e45a04862bd2db529cc, entries=100, sequenceid=145, filesize=21.9 K 2024-12-04T19:15:35,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/B/60166ae1812f4567a6a9f7542e53a72e as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/60166ae1812f4567a6a9f7542e53a72e 2024-12-04T19:15:35,273 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:35,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49134 deadline: 1733339795271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:35,274 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 9ba2734cd39f,33969,1733339629791 for 'test_row_2' in 77154c9b50d6b1163c1785212a1cb410 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-04T19:15:35,274 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:35,274 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:35,274 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:35,276 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/60166ae1812f4567a6a9f7542e53a72e, entries=100, sequenceid=145, filesize=9.5 K 2024-12-04T19:15:35,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/C/74cf0ea3af6d442fb4b0e070a3e01a20 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/74cf0ea3af6d442fb4b0e070a3e01a20 2024-12-04T19:15:35,281 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/74cf0ea3af6d442fb4b0e070a3e01a20, entries=100, sequenceid=145, filesize=9.5 K 2024-12-04T19:15:35,284 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=147.60 KB/151140 for 77154c9b50d6b1163c1785212a1cb410 in 1325ms, sequenceid=145, compaction requested=false 2024-12-04T19:15:35,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for 77154c9b50d6b1163c1785212a1cb410: 2024-12-04T19:15:35,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:35,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-12-04T19:15:35,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-12-04T19:15:35,287 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-12-04T19:15:35,287 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6320 sec 2024-12-04T19:15:35,289 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 1.6360 sec 2024-12-04T19:15:35,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(8855): Flush requested on 77154c9b50d6b1163c1785212a1cb410 2024-12-04T19:15:35,290 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 77154c9b50d6b1163c1785212a1cb410 3/3 column families, dataSize=154.31 KB heapSize=430.69 KB 2024-12-04T19:15:35,290 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77154c9b50d6b1163c1785212a1cb410, store=A 2024-12-04T19:15:35,290 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T19:15:35,290 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77154c9b50d6b1163c1785212a1cb410, store=B 2024-12-04T19:15:35,290 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T19:15:35,290 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77154c9b50d6b1163c1785212a1cb410, store=C 2024-12-04T19:15:35,290 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T19:15:35,297 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204f1bdf7696d704a74a01801cdc989e896_77154c9b50d6b1163c1785212a1cb410 is 50, key is test_row_0/A:col10/1733339735288/Put/seqid=0 2024-12-04T19:15:35,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742202_1378 (size=14794) 2024-12-04T19:15:35,341 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:35,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49042 deadline: 1733339795333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:35,341 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:35,341 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:35,341 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:35,460 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:35,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49042 deadline: 1733339795452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:35,461 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:35,461 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:35,461 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:35,677 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:35,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49042 deadline: 1733339795672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:35,677 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:35,677 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:35,677 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:35,713 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:35,716 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204f1bdf7696d704a74a01801cdc989e896_77154c9b50d6b1163c1785212a1cb410 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204f1bdf7696d704a74a01801cdc989e896_77154c9b50d6b1163c1785212a1cb410 2024-12-04T19:15:35,717 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/A/fe6ee7719f9d464ea9ce565df1ae1cc3, store: [table=TestAcidGuarantees family=A region=77154c9b50d6b1163c1785212a1cb410] 2024-12-04T19:15:35,717 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/A/fe6ee7719f9d464ea9ce565df1ae1cc3 is 175, key is test_row_0/A:col10/1733339735288/Put/seqid=0 2024-12-04T19:15:35,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742203_1379 (size=39749) 2024-12-04T19:15:35,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-04T19:15:35,802 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T19:15:35,802 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T19:15:35,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-12-04T19:15:35,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-04T19:15:35,804 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T19:15:35,804 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T19:15:35,805 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T19:15:35,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-04T19:15:35,956 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-04T19:15:35,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:35,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. as already flushing 2024-12-04T19:15:35,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:35,957 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:35,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:35,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:35,986 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:35,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49042 deadline: 1733339795981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:35,987 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:35,987 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:35,987 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:36,109 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-04T19:15:36,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:36,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. as already flushing 2024-12-04T19:15:36,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:36,109 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:36,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:36,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:36,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-04T19:15:36,127 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=185, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/A/fe6ee7719f9d464ea9ce565df1ae1cc3 2024-12-04T19:15:36,136 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/B/53d275afa3854625930ac048a95d17c3 is 50, key is test_row_0/B:col10/1733339735288/Put/seqid=0 2024-12-04T19:15:36,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742204_1380 (size=12151) 2024-12-04T19:15:36,146 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=185 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/B/53d275afa3854625930ac048a95d17c3 2024-12-04T19:15:36,152 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/C/f81dee1a19334aa9a3508e6de4127930 is 50, key is test_row_0/C:col10/1733339735288/Put/seqid=0 2024-12-04T19:15:36,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742205_1381 (size=12151) 2024-12-04T19:15:36,257 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:36,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49102 deadline: 1733339796254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:36,259 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 9ba2734cd39f,33969,1733339629791 for 'test_row_2' in 77154c9b50d6b1163c1785212a1cb410 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-04T19:15:36,259 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:36,259 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:36,259 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:36,262 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-04T19:15:36,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:36,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. as already flushing 2024-12-04T19:15:36,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:36,262 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:36,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:36,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:36,280 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:36,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49070 deadline: 1733339796272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:36,280 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:36,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49118 deadline: 1733339796272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:36,280 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 9ba2734cd39f,33969,1733339629791 for 'test_row_1' in 77154c9b50d6b1163c1785212a1cb410 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-04T19:15:36,281 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:36,281 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:36,281 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:36,280 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 9ba2734cd39f,33969,1733339629791 for 'test_row_0' in 77154c9b50d6b1163c1785212a1cb410 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-04T19:15:36,281 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:36,281 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:36,281 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:36,415 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-04T19:15:36,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:36,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. as already flushing 2024-12-04T19:15:36,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:36,415 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:36,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:36,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:36,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-04T19:15:36,502 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:36,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49042 deadline: 1733339796494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:36,506 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:36,506 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:36,506 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:36,556 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=185 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/C/f81dee1a19334aa9a3508e6de4127930 2024-12-04T19:15:36,561 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/A/fe6ee7719f9d464ea9ce565df1ae1cc3 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/fe6ee7719f9d464ea9ce565df1ae1cc3 2024-12-04T19:15:36,570 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-04T19:15:36,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:36,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. as already flushing 2024-12-04T19:15:36,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:36,571 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:36,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:36,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:36,573 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/fe6ee7719f9d464ea9ce565df1ae1cc3, entries=200, sequenceid=185, filesize=38.8 K 2024-12-04T19:15:36,574 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/B/53d275afa3854625930ac048a95d17c3 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/53d275afa3854625930ac048a95d17c3 2024-12-04T19:15:36,579 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/53d275afa3854625930ac048a95d17c3, entries=150, sequenceid=185, filesize=11.9 K 2024-12-04T19:15:36,583 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/C/f81dee1a19334aa9a3508e6de4127930 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/f81dee1a19334aa9a3508e6de4127930 2024-12-04T19:15:36,603 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/f81dee1a19334aa9a3508e6de4127930, entries=150, sequenceid=185, filesize=11.9 K 2024-12-04T19:15:36,604 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~430.64 KB/440976, currentSize=33.54 KB/34350 for 77154c9b50d6b1163c1785212a1cb410 in 1315ms, sequenceid=185, compaction requested=true 2024-12-04T19:15:36,604 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 77154c9b50d6b1163c1785212a1cb410: 2024-12-04T19:15:36,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 77154c9b50d6b1163c1785212a1cb410:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T19:15:36,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T19:15:36,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 77154c9b50d6b1163c1785212a1cb410:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T19:15:36,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-04T19:15:36,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 77154c9b50d6b1163c1785212a1cb410:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T19:15:36,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-12-04T19:15:36,605 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T19:15:36,605 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T19:15:36,606 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34299 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T19:15:36,606 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1541): 77154c9b50d6b1163c1785212a1cb410/C is initiating minor compaction (all files) 2024-12-04T19:15:36,606 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 77154c9b50d6b1163c1785212a1cb410/C in TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:36,607 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/c7361166861f48bfbba7bbd4f9db6381, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/74cf0ea3af6d442fb4b0e070a3e01a20, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/f81dee1a19334aa9a3508e6de4127930] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp, totalSize=33.5 K 2024-12-04T19:15:36,607 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93555 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T19:15:36,607 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1541): 77154c9b50d6b1163c1785212a1cb410/A is initiating minor compaction (all files) 2024-12-04T19:15:36,607 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 77154c9b50d6b1163c1785212a1cb410/A in TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:36,607 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/1cea96d199fc422ebf161ecf2a2735c7, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/ecc20430f3924e45a04862bd2db529cc, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/fe6ee7719f9d464ea9ce565df1ae1cc3] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp, totalSize=91.4 K 2024-12-04T19:15:36,607 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:36,607 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. files: [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/1cea96d199fc422ebf161ecf2a2735c7, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/ecc20430f3924e45a04862bd2db529cc, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/fe6ee7719f9d464ea9ce565df1ae1cc3] 2024-12-04T19:15:36,607 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting c7361166861f48bfbba7bbd4f9db6381, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1733339731888 2024-12-04T19:15:36,608 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1cea96d199fc422ebf161ecf2a2735c7, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1733339731888 2024-12-04T19:15:36,608 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting ecc20430f3924e45a04862bd2db529cc, keycount=100, bloomtype=ROW, size=21.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1733339733104 2024-12-04T19:15:36,608 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 74cf0ea3af6d442fb4b0e070a3e01a20, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1733339733104 2024-12-04T19:15:36,608 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting fe6ee7719f9d464ea9ce565df1ae1cc3, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1733339734054 2024-12-04T19:15:36,609 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting f81dee1a19334aa9a3508e6de4127930, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1733339734070 2024-12-04T19:15:36,621 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 77154c9b50d6b1163c1785212a1cb410#C#compaction#323 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:15:36,621 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/C/7c00618b71b849f7b69461b5d688f95e is 50, key is test_row_0/C:col10/1733339735288/Put/seqid=0 2024-12-04T19:15:36,629 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=77154c9b50d6b1163c1785212a1cb410] 2024-12-04T19:15:36,657 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241204d30fba3a038b48558a2d3cab9a6b460f_77154c9b50d6b1163c1785212a1cb410 store=[table=TestAcidGuarantees family=A region=77154c9b50d6b1163c1785212a1cb410] 2024-12-04T19:15:36,658 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241204d30fba3a038b48558a2d3cab9a6b460f_77154c9b50d6b1163c1785212a1cb410, store=[table=TestAcidGuarantees family=A region=77154c9b50d6b1163c1785212a1cb410] 2024-12-04T19:15:36,658 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204d30fba3a038b48558a2d3cab9a6b460f_77154c9b50d6b1163c1785212a1cb410 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=77154c9b50d6b1163c1785212a1cb410] 2024-12-04T19:15:36,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742206_1382 (size=12493) 2024-12-04T19:15:36,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742207_1383 (size=4469) 2024-12-04T19:15:36,725 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-04T19:15:36,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:36,725 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing 77154c9b50d6b1163c1785212a1cb410 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-04T19:15:36,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77154c9b50d6b1163c1785212a1cb410, store=A 2024-12-04T19:15:36,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:15:36,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77154c9b50d6b1163c1785212a1cb410, store=B 2024-12-04T19:15:36,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:15:36,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77154c9b50d6b1163c1785212a1cb410, store=C 2024-12-04T19:15:36,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:15:36,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204a305a3e14f7e4b97a4e5d2171a6ee97f_77154c9b50d6b1163c1785212a1cb410 is 50, key is test_row_0/A:col10/1733339735322/Put/seqid=0 2024-12-04T19:15:36,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742208_1384 (size=12304) 2024-12-04T19:15:36,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-04T19:15:37,077 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 77154c9b50d6b1163c1785212a1cb410#A#compaction#324 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:15:37,078 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/A/c5aba57e26c04f77949f3f11f589fd1d is 175, key is test_row_0/A:col10/1733339735288/Put/seqid=0 2024-12-04T19:15:37,097 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/C/7c00618b71b849f7b69461b5d688f95e as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/7c00618b71b849f7b69461b5d688f95e 2024-12-04T19:15:37,104 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 77154c9b50d6b1163c1785212a1cb410/C of 77154c9b50d6b1163c1785212a1cb410 into 7c00618b71b849f7b69461b5d688f95e(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:15:37,104 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 77154c9b50d6b1163c1785212a1cb410: 2024-12-04T19:15:37,104 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., storeName=77154c9b50d6b1163c1785212a1cb410/C, priority=13, startTime=1733339736605; duration=0sec 2024-12-04T19:15:37,104 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T19:15:37,104 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 77154c9b50d6b1163c1785212a1cb410:C 2024-12-04T19:15:37,104 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T19:15:37,105 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34299 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T19:15:37,105 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1541): 77154c9b50d6b1163c1785212a1cb410/B is initiating minor compaction (all files) 2024-12-04T19:15:37,105 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 77154c9b50d6b1163c1785212a1cb410/B in TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:37,105 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/40b43faf0c7540bb9493721e67ca558b, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/60166ae1812f4567a6a9f7542e53a72e, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/53d275afa3854625930ac048a95d17c3] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp, totalSize=33.5 K 2024-12-04T19:15:37,105 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 40b43faf0c7540bb9493721e67ca558b, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1733339731888 2024-12-04T19:15:37,106 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 60166ae1812f4567a6a9f7542e53a72e, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1733339733104 2024-12-04T19:15:37,106 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 53d275afa3854625930ac048a95d17c3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1733339734070 2024-12-04T19:15:37,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742209_1385 (size=31447) 2024-12-04T19:15:37,162 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 77154c9b50d6b1163c1785212a1cb410#B#compaction#326 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:15:37,163 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/B/719c10a50b5849648dcf5a458077078b is 50, key is test_row_0/B:col10/1733339735288/Put/seqid=0 2024-12-04T19:15:37,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:37,172 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204a305a3e14f7e4b97a4e5d2171a6ee97f_77154c9b50d6b1163c1785212a1cb410 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204a305a3e14f7e4b97a4e5d2171a6ee97f_77154c9b50d6b1163c1785212a1cb410 2024-12-04T19:15:37,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/A/a0a90c211cf042ca90f023610fa6afe4, store: [table=TestAcidGuarantees family=A region=77154c9b50d6b1163c1785212a1cb410] 2024-12-04T19:15:37,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/A/a0a90c211cf042ca90f023610fa6afe4 is 175, key is test_row_0/A:col10/1733339735322/Put/seqid=0 2024-12-04T19:15:37,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742210_1386 (size=12493) 2024-12-04T19:15:37,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742211_1387 (size=31105) 2024-12-04T19:15:37,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(8855): Flush requested on 77154c9b50d6b1163c1785212a1cb410 2024-12-04T19:15:37,293 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. as already flushing 2024-12-04T19:15:37,453 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733339737387, max timestamp=1733339737451Num uniques 859; 2024-12-04T19:15:37,453 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T19:15:37,454 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733339737387, max timestamp=1733339737451Num uniques 859; 2024-12-04T19:15:37,454 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T19:15:37,455 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733339737387, max timestamp=1733339737451Num uniques 859; 2024-12-04T19:15:37,455 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T19:15:37,550 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733339737529, max timestamp=1733339737547Num uniques 1018; 2024-12-04T19:15:37,550 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-04T19:15:37,550 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733339737529, max timestamp=1733339737547Num uniques 1018; 2024-12-04T19:15:37,550 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733339737529, max timestamp=1733339737547Num uniques 1018; 2024-12-04T19:15:37,550 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-04T19:15:37,550 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-04T19:15:37,569 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/A/c5aba57e26c04f77949f3f11f589fd1d as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/c5aba57e26c04f77949f3f11f589fd1d 2024-12-04T19:15:37,575 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 77154c9b50d6b1163c1785212a1cb410/A of 77154c9b50d6b1163c1785212a1cb410 into c5aba57e26c04f77949f3f11f589fd1d(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:15:37,575 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 77154c9b50d6b1163c1785212a1cb410: 2024-12-04T19:15:37,575 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., storeName=77154c9b50d6b1163c1785212a1cb410/A, priority=13, startTime=1733339736605; duration=0sec 2024-12-04T19:15:37,575 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:15:37,575 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 77154c9b50d6b1163c1785212a1cb410:A 2024-12-04T19:15:37,634 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/B/719c10a50b5849648dcf5a458077078b as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/719c10a50b5849648dcf5a458077078b 2024-12-04T19:15:37,638 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 77154c9b50d6b1163c1785212a1cb410/B of 77154c9b50d6b1163c1785212a1cb410 into 719c10a50b5849648dcf5a458077078b(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:15:37,639 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 77154c9b50d6b1163c1785212a1cb410: 2024-12-04T19:15:37,639 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., storeName=77154c9b50d6b1163c1785212a1cb410/B, priority=13, startTime=1733339736605; duration=0sec 2024-12-04T19:15:37,639 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:15:37,639 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 77154c9b50d6b1163c1785212a1cb410:B 2024-12-04T19:15:37,647 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733339737605, max timestamp=1733339737642Num uniques 1027; 2024-12-04T19:15:37,647 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-12-04T19:15:37,647 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733339737605, max timestamp=1733339737642Num uniques 1027; 2024-12-04T19:15:37,647 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-12-04T19:15:37,648 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733339737605, max timestamp=1733339737642Num uniques 1027; 2024-12-04T19:15:37,648 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-12-04T19:15:37,656 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=193, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/A/a0a90c211cf042ca90f023610fa6afe4 2024-12-04T19:15:37,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/B/155d7d4979744abc9ca6e57275404470 is 50, key is test_row_0/B:col10/1733339735322/Put/seqid=0 2024-12-04T19:15:37,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742212_1388 (size=12151) 2024-12-04T19:15:37,698 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/B/155d7d4979744abc9ca6e57275404470 2024-12-04T19:15:37,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/C/a0fd7662bbf04f609067c68bd5911747 is 50, key is test_row_0/C:col10/1733339735322/Put/seqid=0 2024-12-04T19:15:37,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742213_1389 (size=12151) 2024-12-04T19:15:37,712 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/C/a0fd7662bbf04f609067c68bd5911747 2024-12-04T19:15:37,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/A/a0a90c211cf042ca90f023610fa6afe4 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/a0a90c211cf042ca90f023610fa6afe4 2024-12-04T19:15:37,719 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/a0a90c211cf042ca90f023610fa6afe4, entries=150, sequenceid=193, filesize=30.4 K 2024-12-04T19:15:37,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/B/155d7d4979744abc9ca6e57275404470 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/155d7d4979744abc9ca6e57275404470 2024-12-04T19:15:37,728 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/155d7d4979744abc9ca6e57275404470, entries=150, sequenceid=193, filesize=11.9 K 2024-12-04T19:15:37,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/C/a0fd7662bbf04f609067c68bd5911747 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/a0fd7662bbf04f609067c68bd5911747 2024-12-04T19:15:37,741 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/a0fd7662bbf04f609067c68bd5911747, entries=150, sequenceid=193, filesize=11.9 K 2024-12-04T19:15:37,741 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=107.34 KB/109920 for 77154c9b50d6b1163c1785212a1cb410 in 1016ms, sequenceid=193, compaction requested=false 2024-12-04T19:15:37,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for 77154c9b50d6b1163c1785212a1cb410: 2024-12-04T19:15:37,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:37,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-12-04T19:15:37,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-12-04T19:15:37,745 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-12-04T19:15:37,745 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9390 sec 2024-12-04T19:15:37,750 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 1.9460 sec 2024-12-04T19:15:37,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(8855): Flush requested on 77154c9b50d6b1163c1785212a1cb410 2024-12-04T19:15:37,750 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 77154c9b50d6b1163c1785212a1cb410 3/3 column families, dataSize=114.05 KB heapSize=318.84 KB 2024-12-04T19:15:37,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77154c9b50d6b1163c1785212a1cb410, store=A 2024-12-04T19:15:37,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T19:15:37,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77154c9b50d6b1163c1785212a1cb410, store=B 2024-12-04T19:15:37,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T19:15:37,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77154c9b50d6b1163c1785212a1cb410, store=C 2024-12-04T19:15:37,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T19:15:37,777 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412049c51e099a81d402583d9e517809dc47b_77154c9b50d6b1163c1785212a1cb410 is 50, key is test_row_0/A:col10/1733339737745/Put/seqid=0 2024-12-04T19:15:37,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742214_1390 (size=14944) 2024-12-04T19:15:37,792 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:37,797 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412049c51e099a81d402583d9e517809dc47b_77154c9b50d6b1163c1785212a1cb410 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412049c51e099a81d402583d9e517809dc47b_77154c9b50d6b1163c1785212a1cb410 2024-12-04T19:15:37,799 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/A/e4440c1322d1499eb83623c0912d11d2, store: [table=TestAcidGuarantees family=A region=77154c9b50d6b1163c1785212a1cb410] 2024-12-04T19:15:37,800 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/A/e4440c1322d1499eb83623c0912d11d2 is 175, key is test_row_0/A:col10/1733339737745/Put/seqid=0 2024-12-04T19:15:37,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742215_1391 (size=39899) 2024-12-04T19:15:37,830 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:37,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49042 deadline: 1733339797817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:37,830 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:37,830 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:37,830 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:37,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49134 deadline: 1733339797817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:37,830 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:37,831 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:37,831 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:37,831 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:37,946 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:37,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49042 deadline: 1733339797941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:37,947 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:37,947 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:37,947 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:37,947 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:37,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49134 deadline: 1733339797941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:37,947 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:37,947 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:37,947 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:37,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-04T19:15:37,952 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T19:15:37,953 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T19:15:37,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-12-04T19:15:37,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-04T19:15:37,959 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T19:15:37,960 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T19:15:37,960 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T19:15:38,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-04T19:15:38,113 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-04T19:15:38,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:38,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. as already flushing 2024-12-04T19:15:38,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:38,113 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:38,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:38,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:38,159 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:38,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49042 deadline: 1733339798151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:38,159 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:38,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49134 deadline: 1733339798152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:38,160 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:38,160 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:38,160 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:38,160 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:38,160 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:38,160 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:38,212 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=263, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/A/e4440c1322d1499eb83623c0912d11d2 2024-12-04T19:15:38,223 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/B/2453fe6dc3a74561a50ad0af81d751d7 is 50, key is test_row_0/B:col10/1733339737745/Put/seqid=0 2024-12-04T19:15:38,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742216_1392 (size=12251) 2024-12-04T19:15:38,234 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/B/2453fe6dc3a74561a50ad0af81d751d7 2024-12-04T19:15:38,250 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/C/90e2969d8b024e2ab765cfb235639356 is 50, key is test_row_0/C:col10/1733339737745/Put/seqid=0 2024-12-04T19:15:38,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742217_1393 (size=12251) 2024-12-04T19:15:38,258 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/C/90e2969d8b024e2ab765cfb235639356 2024-12-04T19:15:38,262 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/A/e4440c1322d1499eb83623c0912d11d2 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/e4440c1322d1499eb83623c0912d11d2 2024-12-04T19:15:38,266 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/e4440c1322d1499eb83623c0912d11d2, entries=200, sequenceid=263, filesize=39.0 K 2024-12-04T19:15:38,267 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-04T19:15:38,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:38,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. as already flushing 2024-12-04T19:15:38,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:38,268 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:38,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:38,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:38,269 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/B/2453fe6dc3a74561a50ad0af81d751d7 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/2453fe6dc3a74561a50ad0af81d751d7 2024-12-04T19:15:38,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-04T19:15:38,274 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/2453fe6dc3a74561a50ad0af81d751d7, entries=150, sequenceid=263, filesize=12.0 K 2024-12-04T19:15:38,275 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/C/90e2969d8b024e2ab765cfb235639356 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/90e2969d8b024e2ab765cfb235639356 2024-12-04T19:15:38,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,287 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/90e2969d8b024e2ab765cfb235639356, entries=150, sequenceid=263, filesize=12.0 K 2024-12-04T19:15:38,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,288 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~318.80 KB/326448, currentSize=67.09 KB/68700 for 77154c9b50d6b1163c1785212a1cb410 in 538ms, sequenceid=263, compaction requested=true 2024-12-04T19:15:38,288 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 77154c9b50d6b1163c1785212a1cb410: 2024-12-04T19:15:38,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 77154c9b50d6b1163c1785212a1cb410:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T19:15:38,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:15:38,289 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T19:15:38,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 77154c9b50d6b1163c1785212a1cb410:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T19:15:38,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:15:38,289 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T19:15:38,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 77154c9b50d6b1163c1785212a1cb410:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T19:15:38,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T19:15:38,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,290 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102451 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T19:15:38,290 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1541): 77154c9b50d6b1163c1785212a1cb410/A is initiating minor compaction (all files) 2024-12-04T19:15:38,290 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 77154c9b50d6b1163c1785212a1cb410/A in TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:38,290 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/c5aba57e26c04f77949f3f11f589fd1d, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/a0a90c211cf042ca90f023610fa6afe4, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/e4440c1322d1499eb83623c0912d11d2] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp, totalSize=100.0 K 2024-12-04T19:15:38,290 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:38,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,290 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. files: [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/c5aba57e26c04f77949f3f11f589fd1d, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/a0a90c211cf042ca90f023610fa6afe4, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/e4440c1322d1499eb83623c0912d11d2] 2024-12-04T19:15:38,291 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36895 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T19:15:38,291 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1541): 77154c9b50d6b1163c1785212a1cb410/B is initiating minor compaction (all files) 2024-12-04T19:15:38,291 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 77154c9b50d6b1163c1785212a1cb410/B in TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:38,291 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/719c10a50b5849648dcf5a458077078b, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/155d7d4979744abc9ca6e57275404470, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/2453fe6dc3a74561a50ad0af81d751d7] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp, totalSize=36.0 K 2024-12-04T19:15:38,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,291 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 719c10a50b5849648dcf5a458077078b, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1733339734070 2024-12-04T19:15:38,292 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting c5aba57e26c04f77949f3f11f589fd1d, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1733339734070 2024-12-04T19:15:38,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,292 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 155d7d4979744abc9ca6e57275404470, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1733339735310 2024-12-04T19:15:38,292 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting a0a90c211cf042ca90f023610fa6afe4, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1733339735310 2024-12-04T19:15:38,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,292 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 2453fe6dc3a74561a50ad0af81d751d7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1733339737679 2024-12-04T19:15:38,293 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting e4440c1322d1499eb83623c0912d11d2, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1733339737679 2024-12-04T19:15:38,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,303 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=77154c9b50d6b1163c1785212a1cb410] 2024-12-04T19:15:38,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,304 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 77154c9b50d6b1163c1785212a1cb410#B#compaction#332 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:15:38,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,304 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/B/2341f190f80c4cfe8b91290e712cff4e is 50, key is test_row_0/B:col10/1733339737745/Put/seqid=0 2024-12-04T19:15:38,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,313 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241204193bb34b53ba4260b9d01bd13bcadae4_77154c9b50d6b1163c1785212a1cb410 store=[table=TestAcidGuarantees family=A region=77154c9b50d6b1163c1785212a1cb410] 2024-12-04T19:15:38,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,314 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241204193bb34b53ba4260b9d01bd13bcadae4_77154c9b50d6b1163c1785212a1cb410, store=[table=TestAcidGuarantees family=A region=77154c9b50d6b1163c1785212a1cb410] 2024-12-04T19:15:38,315 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204193bb34b53ba4260b9d01bd13bcadae4_77154c9b50d6b1163c1785212a1cb410 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=77154c9b50d6b1163c1785212a1cb410] 2024-12-04T19:15:38,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(8855): Flush requested on 77154c9b50d6b1163c1785212a1cb410 2024-12-04T19:15:38,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,326 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 77154c9b50d6b1163c1785212a1cb410 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-04T19:15:38,326 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77154c9b50d6b1163c1785212a1cb410, store=A 2024-12-04T19:15:38,326 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:15:38,326 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77154c9b50d6b1163c1785212a1cb410, store=B 2024-12-04T19:15:38,326 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:15:38,326 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77154c9b50d6b1163c1785212a1cb410, store=C 2024-12-04T19:15:38,326 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:15:38,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742218_1394 (size=12695) 2024-12-04T19:15:38,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,362 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204f16e4649d97e4974a5541f29c76eb619_77154c9b50d6b1163c1785212a1cb410 is 50, key is test_row_1/A:col10/1733339737813/Put/seqid=0 2024-12-04T19:15:38,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742219_1395 (size=4469) 2024-12-04T19:15:38,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742220_1396 (size=12454) 2024-12-04T19:15:38,382 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:38,386 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204f16e4649d97e4974a5541f29c76eb619_77154c9b50d6b1163c1785212a1cb410 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204f16e4649d97e4974a5541f29c76eb619_77154c9b50d6b1163c1785212a1cb410 2024-12-04T19:15:38,387 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/A/c146e91979b94794863602166a63a165, store: [table=TestAcidGuarantees family=A region=77154c9b50d6b1163c1785212a1cb410] 2024-12-04T19:15:38,388 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/A/c146e91979b94794863602166a63a165 is 175, key is test_row_1/A:col10/1733339737813/Put/seqid=0 2024-12-04T19:15:38,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742221_1397 (size=31251) 2024-12-04T19:15:38,402 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=277, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/A/c146e91979b94794863602166a63a165 2024-12-04T19:15:38,414 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:38,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49118 deadline: 1733339798400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:38,415 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:38,415 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:38,415 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:38,415 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:38,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49070 deadline: 1733339798410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:38,417 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:38,417 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:38,417 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:38,418 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/B/5a37e6185dcd49cdb33cc429184bb0d0 is 50, key is test_row_1/B:col10/1733339737813/Put/seqid=0 2024-12-04T19:15:38,422 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-04T19:15:38,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:38,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. as already flushing 2024-12-04T19:15:38,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:38,423 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:38,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:38,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:38,428 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:38,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49102 deadline: 1733339798417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:38,429 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:38,429 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:38,429 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:38,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742222_1398 (size=9857) 2024-12-04T19:15:38,433 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/B/5a37e6185dcd49cdb33cc429184bb0d0 2024-12-04T19:15:38,443 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/C/d6ba68f6144d4ad99782223825f00e6f is 50, key is test_row_1/C:col10/1733339737813/Put/seqid=0 2024-12-04T19:15:38,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742223_1399 (size=9857) 2024-12-04T19:15:38,478 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:38,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49042 deadline: 1733339798471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:38,478 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:38,478 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:38,478 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:38,478 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:38,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49134 deadline: 1733339798471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:38,479 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:38,479 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:38,479 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:38,527 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:38,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49118 deadline: 1733339798522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:38,528 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:38,528 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:38,528 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:38,528 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:38,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49070 deadline: 1733339798522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:38,528 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:38,528 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:38,528 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:38,537 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:38,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49102 deadline: 1733339798531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:38,537 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:38,537 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:38,537 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:38,575 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-04T19:15:38,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:38,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. as already flushing 2024-12-04T19:15:38,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:38,576 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:38,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:38,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:38,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-04T19:15:38,728 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-04T19:15:38,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:38,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. as already flushing 2024-12-04T19:15:38,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:38,728 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:38,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:38,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:38,735 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:38,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49118 deadline: 1733339798731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:38,735 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:38,735 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:38,735 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:38,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49070 deadline: 1733339798731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:38,735 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:38,735 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:38,735 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:38,735 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:38,738 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/B/2341f190f80c4cfe8b91290e712cff4e as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/2341f190f80c4cfe8b91290e712cff4e 2024-12-04T19:15:38,741 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 77154c9b50d6b1163c1785212a1cb410/B of 77154c9b50d6b1163c1785212a1cb410 into 2341f190f80c4cfe8b91290e712cff4e(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:15:38,742 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 77154c9b50d6b1163c1785212a1cb410: 2024-12-04T19:15:38,742 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., storeName=77154c9b50d6b1163c1785212a1cb410/B, priority=13, startTime=1733339738289; duration=0sec 2024-12-04T19:15:38,742 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T19:15:38,742 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 77154c9b50d6b1163c1785212a1cb410:B 2024-12-04T19:15:38,742 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T19:15:38,743 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36895 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T19:15:38,743 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1541): 77154c9b50d6b1163c1785212a1cb410/C is initiating minor compaction (all files) 2024-12-04T19:15:38,743 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 77154c9b50d6b1163c1785212a1cb410/C in TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:38,743 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/7c00618b71b849f7b69461b5d688f95e, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/a0fd7662bbf04f609067c68bd5911747, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/90e2969d8b024e2ab765cfb235639356] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp, totalSize=36.0 K 2024-12-04T19:15:38,743 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c00618b71b849f7b69461b5d688f95e, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1733339734070 2024-12-04T19:15:38,743 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting a0fd7662bbf04f609067c68bd5911747, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1733339735310 2024-12-04T19:15:38,744 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 90e2969d8b024e2ab765cfb235639356, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1733339737679 2024-12-04T19:15:38,744 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:38,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49102 deadline: 1733339798741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:38,745 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:38,745 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:38,745 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:38,748 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 77154c9b50d6b1163c1785212a1cb410#C#compaction#337 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:15:38,749 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/C/97fc60f723464a6988d46e9f5068fbde is 50, key is test_row_0/C:col10/1733339737745/Put/seqid=0 2024-12-04T19:15:38,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742224_1400 (size=12695) 2024-12-04T19:15:38,755 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/C/97fc60f723464a6988d46e9f5068fbde as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/97fc60f723464a6988d46e9f5068fbde 2024-12-04T19:15:38,759 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 77154c9b50d6b1163c1785212a1cb410/C of 77154c9b50d6b1163c1785212a1cb410 into 97fc60f723464a6988d46e9f5068fbde(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:15:38,759 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 77154c9b50d6b1163c1785212a1cb410: 2024-12-04T19:15:38,759 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., storeName=77154c9b50d6b1163c1785212a1cb410/C, priority=13, startTime=1733339738289; duration=0sec 2024-12-04T19:15:38,759 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:15:38,759 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 77154c9b50d6b1163c1785212a1cb410:C 2024-12-04T19:15:38,770 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 77154c9b50d6b1163c1785212a1cb410#A#compaction#333 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:15:38,771 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/A/a8cdae5c61ff4e4a9d9e0f910f680934 is 175, key is test_row_0/A:col10/1733339737745/Put/seqid=0 2024-12-04T19:15:38,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742225_1401 (size=31649) 2024-12-04T19:15:38,782 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/A/a8cdae5c61ff4e4a9d9e0f910f680934 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/a8cdae5c61ff4e4a9d9e0f910f680934 2024-12-04T19:15:38,786 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 77154c9b50d6b1163c1785212a1cb410/A of 77154c9b50d6b1163c1785212a1cb410 into a8cdae5c61ff4e4a9d9e0f910f680934(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:15:38,786 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 77154c9b50d6b1163c1785212a1cb410: 2024-12-04T19:15:38,786 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., storeName=77154c9b50d6b1163c1785212a1cb410/A, priority=13, startTime=1733339738288; duration=0sec 2024-12-04T19:15:38,786 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:15:38,786 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 77154c9b50d6b1163c1785212a1cb410:A 2024-12-04T19:15:38,859 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/C/d6ba68f6144d4ad99782223825f00e6f 2024-12-04T19:15:38,863 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/A/c146e91979b94794863602166a63a165 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/c146e91979b94794863602166a63a165 2024-12-04T19:15:38,868 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/c146e91979b94794863602166a63a165, entries=150, sequenceid=277, filesize=30.5 K 2024-12-04T19:15:38,870 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/B/5a37e6185dcd49cdb33cc429184bb0d0 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/5a37e6185dcd49cdb33cc429184bb0d0 2024-12-04T19:15:38,873 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/5a37e6185dcd49cdb33cc429184bb0d0, entries=100, sequenceid=277, filesize=9.6 K 2024-12-04T19:15:38,873 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/C/d6ba68f6144d4ad99782223825f00e6f as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/d6ba68f6144d4ad99782223825f00e6f 2024-12-04T19:15:38,876 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/d6ba68f6144d4ad99782223825f00e6f, entries=100, sequenceid=277, filesize=9.6 K 2024-12-04T19:15:38,877 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 77154c9b50d6b1163c1785212a1cb410 in 551ms, sequenceid=277, compaction requested=false 2024-12-04T19:15:38,877 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 77154c9b50d6b1163c1785212a1cb410: 2024-12-04T19:15:38,881 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-04T19:15:38,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:38,881 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing 77154c9b50d6b1163c1785212a1cb410 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-04T19:15:38,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77154c9b50d6b1163c1785212a1cb410, store=A 2024-12-04T19:15:38,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:15:38,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77154c9b50d6b1163c1785212a1cb410, store=B 2024-12-04T19:15:38,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:15:38,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77154c9b50d6b1163c1785212a1cb410, store=C 2024-12-04T19:15:38,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:15:38,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204c375f3c48af645b89335f07ec40ec475_77154c9b50d6b1163c1785212a1cb410 is 50, key is test_row_0/A:col10/1733339738411/Put/seqid=0 2024-12-04T19:15:38,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742226_1402 (size=12454) 2024-12-04T19:15:38,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(8855): Flush requested on 77154c9b50d6b1163c1785212a1cb410 2024-12-04T19:15:38,996 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. as already flushing 2024-12-04T19:15:39,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:39,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49134 deadline: 1733339799044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:39,051 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:39,051 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:39,051 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:39,052 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:39,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49042 deadline: 1733339799045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:39,052 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:39,052 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:39,052 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:39,058 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:39,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49118 deadline: 1733339799050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:39,058 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:39,058 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:39,059 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:39,059 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:39,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49070 deadline: 1733339799050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:39,059 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:39,059 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:39,059 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:39,059 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:39,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49102 deadline: 1733339799051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:39,059 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:39,059 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:39,059 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:39,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-04T19:15:39,169 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:39,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49134 deadline: 1733339799162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:39,169 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:39,169 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:39,169 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:39,169 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:39,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49042 deadline: 1733339799162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:39,170 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:39,170 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:39,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:39,170 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:39,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49118 deadline: 1733339799162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:39,170 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:39,170 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:39,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:39,170 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:39,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49070 deadline: 1733339799162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:39,170 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:39,171 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:39,171 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:39,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:39,331 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204c375f3c48af645b89335f07ec40ec475_77154c9b50d6b1163c1785212a1cb410 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204c375f3c48af645b89335f07ec40ec475_77154c9b50d6b1163c1785212a1cb410 2024-12-04T19:15:39,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/A/c3a14afb61e14403a87f41132358dc15, store: [table=TestAcidGuarantees family=A region=77154c9b50d6b1163c1785212a1cb410] 2024-12-04T19:15:39,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/A/c3a14afb61e14403a87f41132358dc15 is 175, key is test_row_0/A:col10/1733339738411/Put/seqid=0 2024-12-04T19:15:39,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742227_1403 (size=31255) 2024-12-04T19:15:39,347 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=298, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/A/c3a14afb61e14403a87f41132358dc15 2024-12-04T19:15:39,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/B/101a15c649fe4c78bec3034716fa09c6 is 50, key is test_row_0/B:col10/1733339738411/Put/seqid=0 2024-12-04T19:15:39,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742228_1404 (size=12301) 2024-12-04T19:15:39,372 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/B/101a15c649fe4c78bec3034716fa09c6 2024-12-04T19:15:39,384 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:39,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49134 deadline: 1733339799372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:39,393 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:39,393 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:39,393 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:39,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49042 deadline: 1733339799372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:39,393 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:39,393 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:39,393 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:39,393 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:39,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/C/80fe5af98dcc411abd20dd8d147a2b02 is 50, key is test_row_0/C:col10/1733339738411/Put/seqid=0 2024-12-04T19:15:39,402 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:39,402 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:39,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49070 deadline: 1733339799382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:39,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49118 deadline: 1733339799382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:39,403 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:39,403 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:39,403 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:39,405 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:39,405 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:39,405 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:39,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742229_1405 (size=12301) 2024-12-04T19:15:39,425 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/C/80fe5af98dcc411abd20dd8d147a2b02 2024-12-04T19:15:39,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/A/c3a14afb61e14403a87f41132358dc15 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/c3a14afb61e14403a87f41132358dc15 2024-12-04T19:15:39,438 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/c3a14afb61e14403a87f41132358dc15, entries=150, sequenceid=298, filesize=30.5 K 2024-12-04T19:15:39,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/B/101a15c649fe4c78bec3034716fa09c6 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/101a15c649fe4c78bec3034716fa09c6 2024-12-04T19:15:39,446 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/101a15c649fe4c78bec3034716fa09c6, entries=150, sequenceid=298, filesize=12.0 K 2024-12-04T19:15:39,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/C/80fe5af98dcc411abd20dd8d147a2b02 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/80fe5af98dcc411abd20dd8d147a2b02 2024-12-04T19:15:39,457 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/80fe5af98dcc411abd20dd8d147a2b02, entries=150, sequenceid=298, filesize=12.0 K 2024-12-04T19:15:39,458 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 77154c9b50d6b1163c1785212a1cb410 in 577ms, sequenceid=298, compaction requested=true 2024-12-04T19:15:39,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for 77154c9b50d6b1163c1785212a1cb410: 2024-12-04T19:15:39,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:39,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-12-04T19:15:39,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-12-04T19:15:39,463 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-12-04T19:15:39,464 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5000 sec 2024-12-04T19:15:39,466 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 1.5110 sec 2024-12-04T19:15:39,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(8855): Flush requested on 77154c9b50d6b1163c1785212a1cb410 2024-12-04T19:15:39,576 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 77154c9b50d6b1163c1785212a1cb410 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-04T19:15:39,576 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77154c9b50d6b1163c1785212a1cb410, store=A 2024-12-04T19:15:39,576 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:15:39,576 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77154c9b50d6b1163c1785212a1cb410, store=B 2024-12-04T19:15:39,576 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:15:39,576 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77154c9b50d6b1163c1785212a1cb410, store=C 2024-12-04T19:15:39,576 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:15:39,602 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204ff89b53e06d84cd7add79a2fe0f07f56_77154c9b50d6b1163c1785212a1cb410 is 50, key is test_row_0/A:col10/1733339739043/Put/seqid=0 2024-12-04T19:15:39,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742230_1406 (size=14994) 2024-12-04T19:15:39,718 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:39,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49102 deadline: 1733339799711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:39,718 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:39,718 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:39,718 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:39,724 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:39,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49070 deadline: 1733339799711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:39,725 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:39,725 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:39,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49042 deadline: 1733339799717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:39,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49118 deadline: 1733339799712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:39,725 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:39,725 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:39,725 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:39,725 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:39,725 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:39,725 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:39,725 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:39,725 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:39,725 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:39,731 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:39,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49134 deadline: 1733339799718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:39,732 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:39,732 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:39,732 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:39,833 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:39,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49102 deadline: 1733339799822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:39,833 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:39,833 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:39,833 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:39,835 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:39,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49042 deadline: 1733339799832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:39,835 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:39,835 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:39,835 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:39,846 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:39,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49134 deadline: 1733339799841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:39,847 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:39,847 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:39,847 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:40,037 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:40,040 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204ff89b53e06d84cd7add79a2fe0f07f56_77154c9b50d6b1163c1785212a1cb410 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204ff89b53e06d84cd7add79a2fe0f07f56_77154c9b50d6b1163c1785212a1cb410 2024-12-04T19:15:40,041 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/A/96534a0bf8934c4db00ed3731f23d9ae, store: [table=TestAcidGuarantees family=A region=77154c9b50d6b1163c1785212a1cb410] 2024-12-04T19:15:40,042 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/A/96534a0bf8934c4db00ed3731f23d9ae is 175, key is test_row_0/A:col10/1733339739043/Put/seqid=0 2024-12-04T19:15:40,049 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:40,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49102 deadline: 1733339800042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:40,049 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:40,049 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:40,049 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:40,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49042 deadline: 1733339800042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:40,049 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:40,050 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:40,050 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:40,050 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:40,057 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:40,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49134 deadline: 1733339800051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:40,057 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:40,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742231_1407 (size=39949) 2024-12-04T19:15:40,057 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:40,057 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:40,058 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=313, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/A/96534a0bf8934c4db00ed3731f23d9ae 2024-12-04T19:15:40,074 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/B/57ee717f0af74c269ae2bf715dccdf29 is 50, key is test_row_0/B:col10/1733339739043/Put/seqid=0 2024-12-04T19:15:40,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742232_1408 (size=12301) 2024-12-04T19:15:40,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-04T19:15:40,102 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T19:15:40,103 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T19:15:40,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-12-04T19:15:40,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-04T19:15:40,105 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T19:15:40,106 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T19:15:40,106 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T19:15:40,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-04T19:15:40,241 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:40,241 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:40,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49118 deadline: 1733339800232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:40,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49070 deadline: 1733339800232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:40,241 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:40,241 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:40,241 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:40,241 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:40,241 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:40,241 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:40,258 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-04T19:15:40,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:40,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. as already flushing 2024-12-04T19:15:40,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:40,258 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:40,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:40,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:40,362 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:40,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49102 deadline: 1733339800352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:40,363 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:40,363 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:40,363 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:40,369 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:40,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49042 deadline: 1733339800361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:40,369 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:40,369 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:40,369 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:40,369 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:40,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49134 deadline: 1733339800361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:40,370 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:40,370 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:40,370 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:40,411 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-04T19:15:40,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:40,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. as already flushing 2024-12-04T19:15:40,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:40,411 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:40,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:40,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:40,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-04T19:15:40,482 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/B/57ee717f0af74c269ae2bf715dccdf29 2024-12-04T19:15:40,488 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/C/92657249449146fe9121a475d0295ae9 is 50, key is test_row_0/C:col10/1733339739043/Put/seqid=0 2024-12-04T19:15:40,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742233_1409 (size=12301) 2024-12-04T19:15:40,564 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-04T19:15:40,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:40,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. as already flushing 2024-12-04T19:15:40,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:40,565 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:40,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:40,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:40,720 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-04T19:15:40,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:40,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. as already flushing 2024-12-04T19:15:40,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:40,720 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:40,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:40,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:40,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-04T19:15:40,875 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-04T19:15:40,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:40,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. as already flushing 2024-12-04T19:15:40,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:40,876 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:40,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:40,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:40,883 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:40,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49102 deadline: 1733339800873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:40,883 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:40,883 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:40,884 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:40,887 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:40,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49042 deadline: 1733339800882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:40,888 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:40,888 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:40,888 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:40,888 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:40,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49134 deadline: 1733339800882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:40,889 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:40,889 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:40,889 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:40,908 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/C/92657249449146fe9121a475d0295ae9 2024-12-04T19:15:40,914 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/A/96534a0bf8934c4db00ed3731f23d9ae as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/96534a0bf8934c4db00ed3731f23d9ae 2024-12-04T19:15:40,919 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/96534a0bf8934c4db00ed3731f23d9ae, entries=200, sequenceid=313, filesize=39.0 K 2024-12-04T19:15:40,920 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/B/57ee717f0af74c269ae2bf715dccdf29 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/57ee717f0af74c269ae2bf715dccdf29 2024-12-04T19:15:40,924 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/57ee717f0af74c269ae2bf715dccdf29, entries=150, sequenceid=313, filesize=12.0 K 2024-12-04T19:15:40,925 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/C/92657249449146fe9121a475d0295ae9 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/92657249449146fe9121a475d0295ae9 2024-12-04T19:15:40,931 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/92657249449146fe9121a475d0295ae9, entries=150, sequenceid=313, filesize=12.0 K 2024-12-04T19:15:40,933 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for 77154c9b50d6b1163c1785212a1cb410 in 1356ms, sequenceid=313, compaction requested=true 2024-12-04T19:15:40,933 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 77154c9b50d6b1163c1785212a1cb410: 2024-12-04T19:15:40,933 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T19:15:40,935 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 134104 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T19:15:40,935 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1541): 77154c9b50d6b1163c1785212a1cb410/A is initiating minor compaction (all files) 2024-12-04T19:15:40,935 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 77154c9b50d6b1163c1785212a1cb410/A in TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:40,935 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/a8cdae5c61ff4e4a9d9e0f910f680934, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/c146e91979b94794863602166a63a165, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/c3a14afb61e14403a87f41132358dc15, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/96534a0bf8934c4db00ed3731f23d9ae] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp, totalSize=131.0 K 2024-12-04T19:15:40,935 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:40,935 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. files: [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/a8cdae5c61ff4e4a9d9e0f910f680934, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/c146e91979b94794863602166a63a165, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/c3a14afb61e14403a87f41132358dc15, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/96534a0bf8934c4db00ed3731f23d9ae] 2024-12-04T19:15:40,935 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting a8cdae5c61ff4e4a9d9e0f910f680934, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1733339737679 2024-12-04T19:15:40,936 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting c146e91979b94794863602166a63a165, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1733339737782 2024-12-04T19:15:40,936 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting c3a14afb61e14403a87f41132358dc15, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1733339738346 2024-12-04T19:15:40,936 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 96534a0bf8934c4db00ed3731f23d9ae, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1733339739043 2024-12-04T19:15:40,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 77154c9b50d6b1163c1785212a1cb410:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T19:15:40,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:15:40,943 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T19:15:40,944 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47154 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T19:15:40,944 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1541): 77154c9b50d6b1163c1785212a1cb410/B is initiating minor compaction (all files) 2024-12-04T19:15:40,944 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 77154c9b50d6b1163c1785212a1cb410/B in TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:40,944 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/2341f190f80c4cfe8b91290e712cff4e, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/5a37e6185dcd49cdb33cc429184bb0d0, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/101a15c649fe4c78bec3034716fa09c6, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/57ee717f0af74c269ae2bf715dccdf29] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp, totalSize=46.0 K 2024-12-04T19:15:40,945 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 2341f190f80c4cfe8b91290e712cff4e, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1733339737679 2024-12-04T19:15:40,945 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a37e6185dcd49cdb33cc429184bb0d0, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1733339737813 2024-12-04T19:15:40,945 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 101a15c649fe4c78bec3034716fa09c6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1733339738346 2024-12-04T19:15:40,946 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 57ee717f0af74c269ae2bf715dccdf29, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1733339739043 2024-12-04T19:15:40,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 77154c9b50d6b1163c1785212a1cb410:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T19:15:40,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:15:40,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 77154c9b50d6b1163c1785212a1cb410:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T19:15:40,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T19:15:40,958 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=77154c9b50d6b1163c1785212a1cb410] 2024-12-04T19:15:40,972 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 77154c9b50d6b1163c1785212a1cb410#B#compaction#345 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:15:40,972 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/B/2f27d226c58947a08b77a9af70bd66bc is 50, key is test_row_0/B:col10/1733339739043/Put/seqid=0 2024-12-04T19:15:40,981 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241204a0877d46fdb84a2896f4f18b9e22df01_77154c9b50d6b1163c1785212a1cb410 store=[table=TestAcidGuarantees family=A region=77154c9b50d6b1163c1785212a1cb410] 2024-12-04T19:15:40,983 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241204a0877d46fdb84a2896f4f18b9e22df01_77154c9b50d6b1163c1785212a1cb410, store=[table=TestAcidGuarantees family=A region=77154c9b50d6b1163c1785212a1cb410] 2024-12-04T19:15:40,984 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204a0877d46fdb84a2896f4f18b9e22df01_77154c9b50d6b1163c1785212a1cb410 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=77154c9b50d6b1163c1785212a1cb410] 2024-12-04T19:15:41,029 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-04T19:15:41,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:41,030 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing 77154c9b50d6b1163c1785212a1cb410 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-04T19:15:41,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77154c9b50d6b1163c1785212a1cb410, store=A 2024-12-04T19:15:41,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:15:41,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77154c9b50d6b1163c1785212a1cb410, store=B 2024-12-04T19:15:41,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:15:41,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77154c9b50d6b1163c1785212a1cb410, store=C 2024-12-04T19:15:41,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:15:41,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742234_1410 (size=12881) 2024-12-04T19:15:41,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742235_1411 (size=4469) 2024-12-04T19:15:41,095 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 77154c9b50d6b1163c1785212a1cb410#A#compaction#344 average throughput is 0.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:15:41,096 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/A/0565b50961be46798e2cd4ac7defe135 is 175, key is test_row_0/A:col10/1733339739043/Put/seqid=0 2024-12-04T19:15:41,103 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/B/2f27d226c58947a08b77a9af70bd66bc as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/2f27d226c58947a08b77a9af70bd66bc 2024-12-04T19:15:41,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412041158fa1ef4dd47fc8f0e71bceedc52f8_77154c9b50d6b1163c1785212a1cb410 is 50, key is test_row_0/A:col10/1733339739657/Put/seqid=0 2024-12-04T19:15:41,110 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 77154c9b50d6b1163c1785212a1cb410/B of 77154c9b50d6b1163c1785212a1cb410 into 2f27d226c58947a08b77a9af70bd66bc(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:15:41,110 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 77154c9b50d6b1163c1785212a1cb410: 2024-12-04T19:15:41,110 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., storeName=77154c9b50d6b1163c1785212a1cb410/B, priority=12, startTime=1733339740943; duration=0sec 2024-12-04T19:15:41,110 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T19:15:41,110 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 77154c9b50d6b1163c1785212a1cb410:B 2024-12-04T19:15:41,110 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T19:15:41,111 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47154 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T19:15:41,112 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1541): 77154c9b50d6b1163c1785212a1cb410/C is initiating minor compaction (all files) 2024-12-04T19:15:41,112 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 77154c9b50d6b1163c1785212a1cb410/C in TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:41,112 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/97fc60f723464a6988d46e9f5068fbde, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/d6ba68f6144d4ad99782223825f00e6f, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/80fe5af98dcc411abd20dd8d147a2b02, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/92657249449146fe9121a475d0295ae9] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp, totalSize=46.0 K 2024-12-04T19:15:41,112 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 97fc60f723464a6988d46e9f5068fbde, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1733339737679 2024-12-04T19:15:41,112 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting d6ba68f6144d4ad99782223825f00e6f, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1733339737813 2024-12-04T19:15:41,113 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 80fe5af98dcc411abd20dd8d147a2b02, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1733339738346 2024-12-04T19:15:41,113 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 92657249449146fe9121a475d0295ae9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1733339739043 2024-12-04T19:15:41,130 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 77154c9b50d6b1163c1785212a1cb410#C#compaction#347 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:15:41,130 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/C/163ca0b18e8b46568837a97f7cd497e7 is 50, key is test_row_0/C:col10/1733339739043/Put/seqid=0 2024-12-04T19:15:41,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742236_1412 (size=31835) 2024-12-04T19:15:41,146 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/A/0565b50961be46798e2cd4ac7defe135 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/0565b50961be46798e2cd4ac7defe135 2024-12-04T19:15:41,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742237_1413 (size=12454) 2024-12-04T19:15:41,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:41,154 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 77154c9b50d6b1163c1785212a1cb410/A of 77154c9b50d6b1163c1785212a1cb410 into 0565b50961be46798e2cd4ac7defe135(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:15:41,155 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 77154c9b50d6b1163c1785212a1cb410: 2024-12-04T19:15:41,155 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., storeName=77154c9b50d6b1163c1785212a1cb410/A, priority=12, startTime=1733339740933; duration=0sec 2024-12-04T19:15:41,155 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:15:41,155 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 77154c9b50d6b1163c1785212a1cb410:A 2024-12-04T19:15:41,156 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412041158fa1ef4dd47fc8f0e71bceedc52f8_77154c9b50d6b1163c1785212a1cb410 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412041158fa1ef4dd47fc8f0e71bceedc52f8_77154c9b50d6b1163c1785212a1cb410 2024-12-04T19:15:41,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/A/64793b4a956c439d8a405934a862bf1a, store: [table=TestAcidGuarantees family=A region=77154c9b50d6b1163c1785212a1cb410] 2024-12-04T19:15:41,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/A/64793b4a956c439d8a405934a862bf1a is 175, key is test_row_0/A:col10/1733339739657/Put/seqid=0 2024-12-04T19:15:41,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742238_1414 (size=12881) 2024-12-04T19:15:41,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742239_1415 (size=31255) 2024-12-04T19:15:41,174 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=331, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/A/64793b4a956c439d8a405934a862bf1a 2024-12-04T19:15:41,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/B/91617287c87249dea7a002166a74bb53 is 50, key is test_row_0/B:col10/1733339739657/Put/seqid=0 2024-12-04T19:15:41,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742240_1416 (size=12301) 2024-12-04T19:15:41,212 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/B/91617287c87249dea7a002166a74bb53 2024-12-04T19:15:41,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/C/4896ec3b3b5a4c66a8321959377da7e4 is 50, key is test_row_0/C:col10/1733339739657/Put/seqid=0 2024-12-04T19:15:41,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742241_1417 (size=12301) 2024-12-04T19:15:41,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-04T19:15:41,255 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. as already flushing 2024-12-04T19:15:41,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(8855): Flush requested on 77154c9b50d6b1163c1785212a1cb410 2024-12-04T19:15:41,294 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:41,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49118 deadline: 1733339801287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:41,294 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:41,294 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:41,294 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:41,295 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:41,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49070 deadline: 1733339801291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:41,295 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:41,295 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:41,295 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:41,406 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:41,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49118 deadline: 1733339801401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:41,407 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:41,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49070 deadline: 1733339801402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:41,407 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:41,407 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:41,407 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:41,407 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:41,407 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:41,407 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:41,575 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/C/163ca0b18e8b46568837a97f7cd497e7 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/163ca0b18e8b46568837a97f7cd497e7 2024-12-04T19:15:41,581 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 77154c9b50d6b1163c1785212a1cb410/C of 77154c9b50d6b1163c1785212a1cb410 into 163ca0b18e8b46568837a97f7cd497e7(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:15:41,581 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 77154c9b50d6b1163c1785212a1cb410: 2024-12-04T19:15:41,581 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., storeName=77154c9b50d6b1163c1785212a1cb410/C, priority=12, startTime=1733339740946; duration=0sec 2024-12-04T19:15:41,581 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:15:41,581 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 77154c9b50d6b1163c1785212a1cb410:C 2024-12-04T19:15:41,614 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:41,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49118 deadline: 1733339801611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:41,614 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:41,614 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:41,614 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:41,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49070 deadline: 1733339801611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:41,614 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:41,615 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:41,615 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:41,615 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:41,634 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/C/4896ec3b3b5a4c66a8321959377da7e4 2024-12-04T19:15:41,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/A/64793b4a956c439d8a405934a862bf1a as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/64793b4a956c439d8a405934a862bf1a 2024-12-04T19:15:41,641 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/64793b4a956c439d8a405934a862bf1a, entries=150, sequenceid=331, filesize=30.5 K 2024-12-04T19:15:41,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/B/91617287c87249dea7a002166a74bb53 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/91617287c87249dea7a002166a74bb53 2024-12-04T19:15:41,644 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/91617287c87249dea7a002166a74bb53, entries=150, sequenceid=331, filesize=12.0 K 2024-12-04T19:15:41,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/C/4896ec3b3b5a4c66a8321959377da7e4 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/4896ec3b3b5a4c66a8321959377da7e4 2024-12-04T19:15:41,648 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/4896ec3b3b5a4c66a8321959377da7e4, entries=150, sequenceid=331, filesize=12.0 K 2024-12-04T19:15:41,649 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 77154c9b50d6b1163c1785212a1cb410 in 620ms, sequenceid=331, compaction requested=false 2024-12-04T19:15:41,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for 77154c9b50d6b1163c1785212a1cb410: 2024-12-04T19:15:41,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:41,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-12-04T19:15:41,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-12-04T19:15:41,652 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-12-04T19:15:41,652 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5440 sec 2024-12-04T19:15:41,654 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 1.5500 sec 2024-12-04T19:15:41,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(8855): Flush requested on 77154c9b50d6b1163c1785212a1cb410 2024-12-04T19:15:41,897 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 77154c9b50d6b1163c1785212a1cb410 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-04T19:15:41,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77154c9b50d6b1163c1785212a1cb410, store=A 2024-12-04T19:15:41,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:15:41,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77154c9b50d6b1163c1785212a1cb410, store=B 2024-12-04T19:15:41,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:15:41,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77154c9b50d6b1163c1785212a1cb410, store=C 2024-12-04T19:15:41,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:15:41,903 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204dc56acab3dd741beac96aae10318cd59_77154c9b50d6b1163c1785212a1cb410 is 50, key is test_row_0/A:col10/1733339741286/Put/seqid=0 2024-12-04T19:15:41,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742242_1418 (size=14994) 2024-12-04T19:15:41,934 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:41,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49102 deadline: 1733339801924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:41,934 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:41,934 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:41,934 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:41,938 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:41,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49042 deadline: 1733339801932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:41,939 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:41,939 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:41,939 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:41,945 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:41,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49118 deadline: 1733339801933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:41,946 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:41,946 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:41,946 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:41,946 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:41,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49070 deadline: 1733339801933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:41,946 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:41,946 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:41,946 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:41,946 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:41,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49134 deadline: 1733339801934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:41,946 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:41,946 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:41,946 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:42,047 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:42,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49102 deadline: 1733339802042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:42,047 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:42,047 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:42,047 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:42,047 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:42,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49042 deadline: 1733339802042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:42,048 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:42,048 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:42,048 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:42,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:42,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49118 deadline: 1733339802051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:42,056 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:42,056 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:42,056 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:42,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49070 deadline: 1733339802051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:42,056 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:42,056 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:42,056 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:42,056 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:42,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49134 deadline: 1733339802052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:42,056 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:42,056 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:42,056 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:42,056 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:42,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-04T19:15:42,253 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T19:15:42,253 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T19:15:42,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-12-04T19:15:42,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-04T19:15:42,258 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T19:15:42,258 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T19:15:42,259 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T19:15:42,262 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:42,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49042 deadline: 1733339802252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:42,262 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:42,262 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:42,262 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:42,262 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:42,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49102 deadline: 1733339802252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:42,263 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:42,263 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:42,263 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:42,270 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:42,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49118 deadline: 1733339802262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:42,271 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:42,271 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:42,271 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:42,271 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:42,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49070 deadline: 1733339802262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:42,271 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:42,271 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:42,271 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:42,271 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:42,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49134 deadline: 1733339802262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:42,271 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:42,271 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:42,271 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:42,307 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:42,310 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204dc56acab3dd741beac96aae10318cd59_77154c9b50d6b1163c1785212a1cb410 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204dc56acab3dd741beac96aae10318cd59_77154c9b50d6b1163c1785212a1cb410 2024-12-04T19:15:42,311 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/A/9cfba6d3dcc144549339d816380a0627, store: [table=TestAcidGuarantees family=A region=77154c9b50d6b1163c1785212a1cb410] 2024-12-04T19:15:42,311 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/A/9cfba6d3dcc144549339d816380a0627 is 175, key is test_row_0/A:col10/1733339741286/Put/seqid=0 2024-12-04T19:15:42,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742243_1419 (size=39949) 2024-12-04T19:15:42,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-04T19:15:42,410 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-04T19:15:42,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:42,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. as already flushing 2024-12-04T19:15:42,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:42,411 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:42,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:42,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:42,564 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-04T19:15:42,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:42,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. as already flushing 2024-12-04T19:15:42,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:42,565 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:42,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:42,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:42,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-04T19:15:42,576 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:42,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49102 deadline: 1733339802573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:42,579 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:42,579 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:42,579 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:42,583 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:42,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49042 deadline: 1733339802574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:42,583 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:42,583 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:42,583 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:42,588 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:42,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49118 deadline: 1733339802581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:42,588 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:42,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49070 deadline: 1733339802582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:42,589 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:42,589 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:42,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49134 deadline: 1733339802582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:42,589 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:42,589 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:42,589 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:42,589 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:42,589 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:42,589 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:42,589 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:42,589 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:42,714 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=349, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/A/9cfba6d3dcc144549339d816380a0627 2024-12-04T19:15:42,716 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-04T19:15:42,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:42,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. as already flushing 2024-12-04T19:15:42,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:42,717 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:42,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:42,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:42,719 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/B/1bf31b6eac194d8893610f519304ed5c is 50, key is test_row_0/B:col10/1733339741286/Put/seqid=0 2024-12-04T19:15:42,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742244_1420 (size=12301) 2024-12-04T19:15:42,869 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-04T19:15:42,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:42,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. as already flushing 2024-12-04T19:15:42,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:42,869 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:42,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:42,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:42,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-04T19:15:43,022 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-04T19:15:43,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:43,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. as already flushing 2024-12-04T19:15:43,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:43,022 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:43,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:43,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:43,086 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:43,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49102 deadline: 1733339803081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:43,086 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:43,087 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:43,087 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:43,094 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:43,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49042 deadline: 1733339803092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:43,095 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:43,095 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:43,095 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:43,108 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:43,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49070 deadline: 1733339803102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:43,108 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:43,108 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:43,109 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:43,109 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:43,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49134 deadline: 1733339803102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:43,109 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:43,109 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:43,109 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:43,109 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:43,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49118 deadline: 1733339803102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:43,109 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:43,109 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:43,109 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:43,123 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=349 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/B/1bf31b6eac194d8893610f519304ed5c 2024-12-04T19:15:43,130 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/C/269c7e8313b64f1fa03a05d8cd80be79 is 50, key is test_row_0/C:col10/1733339741286/Put/seqid=0 2024-12-04T19:15:43,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742245_1421 (size=12301) 2024-12-04T19:15:43,174 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-04T19:15:43,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:43,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. as already flushing 2024-12-04T19:15:43,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:43,175 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:43,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:43,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:43,327 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-04T19:15:43,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:43,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. as already flushing 2024-12-04T19:15:43,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:43,328 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:43,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:43,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:43,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-04T19:15:43,480 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-04T19:15:43,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:43,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. as already flushing 2024-12-04T19:15:43,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:43,480 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:43,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:43,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:43,542 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=349 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/C/269c7e8313b64f1fa03a05d8cd80be79 2024-12-04T19:15:43,546 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/A/9cfba6d3dcc144549339d816380a0627 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/9cfba6d3dcc144549339d816380a0627 2024-12-04T19:15:43,550 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/9cfba6d3dcc144549339d816380a0627, entries=200, sequenceid=349, filesize=39.0 K 2024-12-04T19:15:43,551 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/B/1bf31b6eac194d8893610f519304ed5c as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/1bf31b6eac194d8893610f519304ed5c 2024-12-04T19:15:43,556 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/1bf31b6eac194d8893610f519304ed5c, entries=150, sequenceid=349, filesize=12.0 K 2024-12-04T19:15:43,557 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/C/269c7e8313b64f1fa03a05d8cd80be79 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/269c7e8313b64f1fa03a05d8cd80be79 2024-12-04T19:15:43,560 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/269c7e8313b64f1fa03a05d8cd80be79, entries=150, sequenceid=349, filesize=12.0 K 2024-12-04T19:15:43,561 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 77154c9b50d6b1163c1785212a1cb410 in 1664ms, sequenceid=349, compaction requested=true 2024-12-04T19:15:43,561 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 77154c9b50d6b1163c1785212a1cb410: 2024-12-04T19:15:43,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 77154c9b50d6b1163c1785212a1cb410:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T19:15:43,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:15:43,562 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T19:15:43,562 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T19:15:43,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 77154c9b50d6b1163c1785212a1cb410:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T19:15:43,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:15:43,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 77154c9b50d6b1163c1785212a1cb410:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T19:15:43,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T19:15:43,562 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T19:15:43,562 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103039 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T19:15:43,563 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1541): 77154c9b50d6b1163c1785212a1cb410/A is initiating minor compaction (all files) 2024-12-04T19:15:43,563 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1541): 77154c9b50d6b1163c1785212a1cb410/B is initiating minor compaction (all files) 2024-12-04T19:15:43,563 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 77154c9b50d6b1163c1785212a1cb410/B in TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:43,563 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 77154c9b50d6b1163c1785212a1cb410/A in TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:43,563 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/0565b50961be46798e2cd4ac7defe135, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/64793b4a956c439d8a405934a862bf1a, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/9cfba6d3dcc144549339d816380a0627] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp, totalSize=100.6 K 2024-12-04T19:15:43,563 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/2f27d226c58947a08b77a9af70bd66bc, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/91617287c87249dea7a002166a74bb53, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/1bf31b6eac194d8893610f519304ed5c] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp, totalSize=36.6 K 2024-12-04T19:15:43,563 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:43,563 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. files: [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/0565b50961be46798e2cd4ac7defe135, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/64793b4a956c439d8a405934a862bf1a, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/9cfba6d3dcc144549339d816380a0627] 2024-12-04T19:15:43,563 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f27d226c58947a08b77a9af70bd66bc, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1733339739043 2024-12-04T19:15:43,563 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0565b50961be46798e2cd4ac7defe135, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1733339739043 2024-12-04T19:15:43,563 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 91617287c87249dea7a002166a74bb53, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1733339739657 2024-12-04T19:15:43,563 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 64793b4a956c439d8a405934a862bf1a, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1733339739657 2024-12-04T19:15:43,563 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 1bf31b6eac194d8893610f519304ed5c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1733339741265 2024-12-04T19:15:43,564 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9cfba6d3dcc144549339d816380a0627, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1733339741265 2024-12-04T19:15:43,574 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=77154c9b50d6b1163c1785212a1cb410] 2024-12-04T19:15:43,576 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 77154c9b50d6b1163c1785212a1cb410#B#compaction#353 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:15:43,576 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/B/69fdbbbfbdb04ebbb9980fd63bfca030 is 50, key is test_row_0/B:col10/1733339741286/Put/seqid=0 2024-12-04T19:15:43,583 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412041b8d2f7549d7431487c3b4f886712a84_77154c9b50d6b1163c1785212a1cb410 store=[table=TestAcidGuarantees family=A region=77154c9b50d6b1163c1785212a1cb410] 2024-12-04T19:15:43,585 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412041b8d2f7549d7431487c3b4f886712a84_77154c9b50d6b1163c1785212a1cb410, store=[table=TestAcidGuarantees family=A region=77154c9b50d6b1163c1785212a1cb410] 2024-12-04T19:15:43,585 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412041b8d2f7549d7431487c3b4f886712a84_77154c9b50d6b1163c1785212a1cb410 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=77154c9b50d6b1163c1785212a1cb410] 2024-12-04T19:15:43,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742246_1422 (size=12983) 2024-12-04T19:15:43,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742247_1423 (size=4469) 2024-12-04T19:15:43,602 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 77154c9b50d6b1163c1785212a1cb410#A#compaction#354 average throughput is 0.87 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:15:43,603 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/A/4e9535e87756459390ed89ad1b12c629 is 175, key is test_row_0/A:col10/1733339741286/Put/seqid=0 2024-12-04T19:15:43,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742248_1424 (size=31937) 2024-12-04T19:15:43,624 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/A/4e9535e87756459390ed89ad1b12c629 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/4e9535e87756459390ed89ad1b12c629 2024-12-04T19:15:43,629 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 77154c9b50d6b1163c1785212a1cb410/A of 77154c9b50d6b1163c1785212a1cb410 into 4e9535e87756459390ed89ad1b12c629(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:15:43,629 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 77154c9b50d6b1163c1785212a1cb410: 2024-12-04T19:15:43,629 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., storeName=77154c9b50d6b1163c1785212a1cb410/A, priority=13, startTime=1733339743561; duration=0sec 2024-12-04T19:15:43,629 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T19:15:43,629 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 77154c9b50d6b1163c1785212a1cb410:A 2024-12-04T19:15:43,629 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T19:15:43,629 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T19:15:43,629 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1541): 77154c9b50d6b1163c1785212a1cb410/C is initiating minor compaction (all files) 2024-12-04T19:15:43,630 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 77154c9b50d6b1163c1785212a1cb410/C in TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:43,630 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/163ca0b18e8b46568837a97f7cd497e7, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/4896ec3b3b5a4c66a8321959377da7e4, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/269c7e8313b64f1fa03a05d8cd80be79] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp, totalSize=36.6 K 2024-12-04T19:15:43,630 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 163ca0b18e8b46568837a97f7cd497e7, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1733339739043 2024-12-04T19:15:43,631 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4896ec3b3b5a4c66a8321959377da7e4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1733339739657 2024-12-04T19:15:43,631 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 269c7e8313b64f1fa03a05d8cd80be79, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1733339741265 2024-12-04T19:15:43,633 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-04T19:15:43,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:43,633 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing 77154c9b50d6b1163c1785212a1cb410 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-04T19:15:43,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77154c9b50d6b1163c1785212a1cb410, store=A 2024-12-04T19:15:43,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:15:43,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77154c9b50d6b1163c1785212a1cb410, store=B 2024-12-04T19:15:43,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:15:43,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77154c9b50d6b1163c1785212a1cb410, store=C 2024-12-04T19:15:43,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:15:43,648 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 77154c9b50d6b1163c1785212a1cb410#C#compaction#355 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:15:43,648 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/C/2e41703bbaeb4fac81790d73df7094f8 is 50, key is test_row_0/C:col10/1733339741286/Put/seqid=0 2024-12-04T19:15:43,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204a8ac695a70424090b8078a5ef5dc1843_77154c9b50d6b1163c1785212a1cb410 is 50, key is test_row_0/A:col10/1733339741923/Put/seqid=0 2024-12-04T19:15:43,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742249_1425 (size=12983) 2024-12-04T19:15:43,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742250_1426 (size=12454) 2024-12-04T19:15:43,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:43,681 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/C/2e41703bbaeb4fac81790d73df7094f8 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/2e41703bbaeb4fac81790d73df7094f8 2024-12-04T19:15:43,682 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204a8ac695a70424090b8078a5ef5dc1843_77154c9b50d6b1163c1785212a1cb410 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204a8ac695a70424090b8078a5ef5dc1843_77154c9b50d6b1163c1785212a1cb410 2024-12-04T19:15:43,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/A/0f4ae009ea9642618a592ba20133bed6, store: [table=TestAcidGuarantees family=A region=77154c9b50d6b1163c1785212a1cb410] 2024-12-04T19:15:43,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/A/0f4ae009ea9642618a592ba20133bed6 is 175, key is test_row_0/A:col10/1733339741923/Put/seqid=0 2024-12-04T19:15:43,691 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 77154c9b50d6b1163c1785212a1cb410/C of 77154c9b50d6b1163c1785212a1cb410 into 2e41703bbaeb4fac81790d73df7094f8(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:15:43,691 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 77154c9b50d6b1163c1785212a1cb410: 2024-12-04T19:15:43,691 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., storeName=77154c9b50d6b1163c1785212a1cb410/C, priority=13, startTime=1733339743562; duration=0sec 2024-12-04T19:15:43,691 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:15:43,691 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 77154c9b50d6b1163c1785212a1cb410:C 2024-12-04T19:15:43,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742251_1427 (size=31255) 2024-12-04T19:15:43,705 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=367, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/A/0f4ae009ea9642618a592ba20133bed6 2024-12-04T19:15:43,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/B/dca1e5524cac49af9e38e5d6f6c7fdac is 50, key is test_row_0/B:col10/1733339741923/Put/seqid=0 2024-12-04T19:15:43,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742252_1428 (size=12301) 2024-12-04T19:15:43,998 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/B/69fdbbbfbdb04ebbb9980fd63bfca030 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/69fdbbbfbdb04ebbb9980fd63bfca030 2024-12-04T19:15:44,005 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 77154c9b50d6b1163c1785212a1cb410/B of 77154c9b50d6b1163c1785212a1cb410 into 69fdbbbfbdb04ebbb9980fd63bfca030(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:15:44,005 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 77154c9b50d6b1163c1785212a1cb410: 2024-12-04T19:15:44,005 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., storeName=77154c9b50d6b1163c1785212a1cb410/B, priority=13, startTime=1733339743562; duration=0sec 2024-12-04T19:15:44,005 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:15:44,005 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 77154c9b50d6b1163c1785212a1cb410:B 2024-12-04T19:15:44,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(8855): Flush requested on 77154c9b50d6b1163c1785212a1cb410 2024-12-04T19:15:44,103 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. as already flushing 2024-12-04T19:15:44,131 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/B/dca1e5524cac49af9e38e5d6f6c7fdac 2024-12-04T19:15:44,137 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:44,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49070 deadline: 1733339804129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:44,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:44,138 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:44,138 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:44,138 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:44,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49118 deadline: 1733339804130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:44,138 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:44,138 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:44,138 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:44,141 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:44,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49102 deadline: 1733339804132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:44,142 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:44,142 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:44,142 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:44,145 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:44,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49134 deadline: 1733339804135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:44,146 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:44,146 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:44,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49042 deadline: 1733339804141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:44,146 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:44,146 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:44,148 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:44,148 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:44,148 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:44,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/C/aba22f12c89f4dd9900972e0d7b810c7 is 50, key is test_row_0/C:col10/1733339741923/Put/seqid=0 2024-12-04T19:15:44,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742253_1429 (size=12301) 2024-12-04T19:15:44,162 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/C/aba22f12c89f4dd9900972e0d7b810c7 2024-12-04T19:15:44,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/A/0f4ae009ea9642618a592ba20133bed6 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/0f4ae009ea9642618a592ba20133bed6 2024-12-04T19:15:44,170 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/0f4ae009ea9642618a592ba20133bed6, entries=150, sequenceid=367, filesize=30.5 K 2024-12-04T19:15:44,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/B/dca1e5524cac49af9e38e5d6f6c7fdac as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/dca1e5524cac49af9e38e5d6f6c7fdac 2024-12-04T19:15:44,176 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/dca1e5524cac49af9e38e5d6f6c7fdac, entries=150, sequenceid=367, filesize=12.0 K 2024-12-04T19:15:44,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/C/aba22f12c89f4dd9900972e0d7b810c7 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/aba22f12c89f4dd9900972e0d7b810c7 2024-12-04T19:15:44,180 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/aba22f12c89f4dd9900972e0d7b810c7, entries=150, sequenceid=367, filesize=12.0 K 2024-12-04T19:15:44,181 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 77154c9b50d6b1163c1785212a1cb410 in 548ms, sequenceid=367, compaction requested=false 2024-12-04T19:15:44,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for 77154c9b50d6b1163c1785212a1cb410: 2024-12-04T19:15:44,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:44,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-12-04T19:15:44,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-12-04T19:15:44,185 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-12-04T19:15:44,185 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9240 sec 2024-12-04T19:15:44,186 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 1.9320 sec 2024-12-04T19:15:44,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(8855): Flush requested on 77154c9b50d6b1163c1785212a1cb410 2024-12-04T19:15:44,247 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 77154c9b50d6b1163c1785212a1cb410 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-04T19:15:44,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77154c9b50d6b1163c1785212a1cb410, store=A 2024-12-04T19:15:44,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:15:44,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77154c9b50d6b1163c1785212a1cb410, store=B 2024-12-04T19:15:44,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:15:44,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77154c9b50d6b1163c1785212a1cb410, store=C 2024-12-04T19:15:44,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:15:44,257 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120489e62b609c6b471da28d2695a807095a_77154c9b50d6b1163c1785212a1cb410 is 50, key is test_row_0/A:col10/1733339744246/Put/seqid=0 2024-12-04T19:15:44,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742254_1430 (size=14994) 2024-12-04T19:15:44,280 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:44,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49134 deadline: 1733339804271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:44,280 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:44,280 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:44,280 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:44,280 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:44,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49042 deadline: 1733339804272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:44,280 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:44,280 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:44,280 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:44,281 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:44,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49102 deadline: 1733339804272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:44,281 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:44,281 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:44,281 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:44,281 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:44,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49070 deadline: 1733339804277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:44,281 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:44,281 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:44,281 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:44,288 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:44,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49118 deadline: 1733339804278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:44,289 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:44,289 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:44,289 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:44,397 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:44,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49134 deadline: 1733339804392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:44,398 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:44,398 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:44,398 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:44,398 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:44,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49102 deadline: 1733339804392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:44,398 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:44,398 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:44,398 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:44,398 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:44,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49042 deadline: 1733339804392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:44,399 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:44,399 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:44,399 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:44,399 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:44,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49070 deadline: 1733339804393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:44,399 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:44,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49118 deadline: 1733339804393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:44,399 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:44,399 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:44,399 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:44,399 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:44,400 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:44,400 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:44,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-04T19:15:44,402 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T19:15:44,402 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T19:15:44,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-12-04T19:15:44,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-04T19:15:44,404 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T19:15:44,404 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T19:15:44,404 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T19:15:44,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-04T19:15:44,556 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-12-04T19:15:44,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:44,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. as already flushing 2024-12-04T19:15:44,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:44,556 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:44,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:44,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:44,607 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:44,607 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:44,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49134 deadline: 1733339804601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:44,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49102 deadline: 1733339804602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:44,607 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:44,607 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:44,607 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:44,607 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:44,608 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:44,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49070 deadline: 1733339804602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:44,608 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:44,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49042 deadline: 1733339804602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:44,608 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:44,608 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:44,608 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:44,608 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:44,608 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:44,608 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:44,608 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:44,608 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:44,616 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:44,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49118 deadline: 1733339804612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:44,616 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:44,616 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:44,617 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:44,674 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:44,676 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120489e62b609c6b471da28d2695a807095a_77154c9b50d6b1163c1785212a1cb410 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120489e62b609c6b471da28d2695a807095a_77154c9b50d6b1163c1785212a1cb410 2024-12-04T19:15:44,677 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/A/56ffe4ffafbd4d6bb708440dd589cc71, store: [table=TestAcidGuarantees family=A region=77154c9b50d6b1163c1785212a1cb410] 2024-12-04T19:15:44,678 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/A/56ffe4ffafbd4d6bb708440dd589cc71 is 175, key is test_row_0/A:col10/1733339744246/Put/seqid=0 2024-12-04T19:15:44,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742255_1431 (size=39949) 2024-12-04T19:15:44,709 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-12-04T19:15:44,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:44,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. as already flushing 2024-12-04T19:15:44,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:44,709 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:44,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:44,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:44,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-04T19:15:44,860 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-12-04T19:15:44,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:44,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. as already flushing 2024-12-04T19:15:44,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:44,861 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:44,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:44,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:44,920 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:44,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49134 deadline: 1733339804911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:44,920 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:44,920 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:44,920 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:44,920 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:44,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49102 deadline: 1733339804911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:44,920 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:44,920 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:44,920 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:44,920 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:44,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49042 deadline: 1733339804911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:44,921 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:44,921 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:44,921 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:44,921 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:44,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49070 deadline: 1733339804912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:44,921 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:44,921 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:44,921 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:44,932 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:44,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49118 deadline: 1733339804922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:44,932 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:44,932 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:44,932 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:15:45,013 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-12-04T19:15:45,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:45,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. as already flushing 2024-12-04T19:15:45,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:45,014 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:45,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:45,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:45,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-04T19:15:45,081 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=385, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/A/56ffe4ffafbd4d6bb708440dd589cc71 2024-12-04T19:15:45,086 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/B/67e78ffae6a04829a138343d254aeb11 is 50, key is test_row_0/B:col10/1733339744246/Put/seqid=0 2024-12-04T19:15:45,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742256_1432 (size=12301) 2024-12-04T19:15:45,093 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=385 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/B/67e78ffae6a04829a138343d254aeb11 2024-12-04T19:15:45,102 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/C/30cad4ba4bee4e5690d632e6008bf925 is 50, key is test_row_0/C:col10/1733339744246/Put/seqid=0 2024-12-04T19:15:45,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742257_1433 (size=12301) 2024-12-04T19:15:45,109 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=385 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/C/30cad4ba4bee4e5690d632e6008bf925 2024-12-04T19:15:45,117 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/A/56ffe4ffafbd4d6bb708440dd589cc71 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/56ffe4ffafbd4d6bb708440dd589cc71 2024-12-04T19:15:45,120 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/56ffe4ffafbd4d6bb708440dd589cc71, entries=200, sequenceid=385, filesize=39.0 K 2024-12-04T19:15:45,120 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/B/67e78ffae6a04829a138343d254aeb11 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/67e78ffae6a04829a138343d254aeb11 2024-12-04T19:15:45,124 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/67e78ffae6a04829a138343d254aeb11, entries=150, sequenceid=385, filesize=12.0 K 2024-12-04T19:15:45,125 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/C/30cad4ba4bee4e5690d632e6008bf925 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/30cad4ba4bee4e5690d632e6008bf925 2024-12-04T19:15:45,130 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/30cad4ba4bee4e5690d632e6008bf925, entries=150, sequenceid=385, filesize=12.0 K 2024-12-04T19:15:45,130 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 77154c9b50d6b1163c1785212a1cb410 in 883ms, sequenceid=385, compaction requested=true 2024-12-04T19:15:45,130 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 77154c9b50d6b1163c1785212a1cb410: 2024-12-04T19:15:45,130 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T19:15:45,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 77154c9b50d6b1163c1785212a1cb410:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T19:15:45,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:15:45,131 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T19:15:45,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 77154c9b50d6b1163c1785212a1cb410:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T19:15:45,131 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T19:15:45,131 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1541): 77154c9b50d6b1163c1785212a1cb410/A is initiating minor compaction (all files) 2024-12-04T19:15:45,131 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 77154c9b50d6b1163c1785212a1cb410/A in TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:45,132 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/4e9535e87756459390ed89ad1b12c629, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/0f4ae009ea9642618a592ba20133bed6, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/56ffe4ffafbd4d6bb708440dd589cc71] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp, totalSize=100.7 K 2024-12-04T19:15:45,132 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:45,132 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. files: [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/4e9535e87756459390ed89ad1b12c629, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/0f4ae009ea9642618a592ba20133bed6, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/56ffe4ffafbd4d6bb708440dd589cc71] 2024-12-04T19:15:45,132 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e9535e87756459390ed89ad1b12c629, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1733339741265 2024-12-04T19:15:45,132 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T19:15:45,132 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1541): 77154c9b50d6b1163c1785212a1cb410/B is initiating minor compaction (all files) 2024-12-04T19:15:45,132 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 77154c9b50d6b1163c1785212a1cb410/B in TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:45,132 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/69fdbbbfbdb04ebbb9980fd63bfca030, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/dca1e5524cac49af9e38e5d6f6c7fdac, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/67e78ffae6a04829a138343d254aeb11] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp, totalSize=36.7 K 2024-12-04T19:15:45,133 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0f4ae009ea9642618a592ba20133bed6, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1733339741923 2024-12-04T19:15:45,133 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 69fdbbbfbdb04ebbb9980fd63bfca030, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1733339741265 2024-12-04T19:15:45,133 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 56ffe4ffafbd4d6bb708440dd589cc71, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1733339744128 2024-12-04T19:15:45,133 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting dca1e5524cac49af9e38e5d6f6c7fdac, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1733339741923 2024-12-04T19:15:45,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:15:45,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 77154c9b50d6b1163c1785212a1cb410:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T19:15:45,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T19:15:45,134 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 67e78ffae6a04829a138343d254aeb11, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1733339744131 2024-12-04T19:15:45,144 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=77154c9b50d6b1163c1785212a1cb410] 2024-12-04T19:15:45,144 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 77154c9b50d6b1163c1785212a1cb410#B#compaction#362 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:15:45,145 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/B/8927d5e142f341bf9090c41a5390cb59 is 50, key is test_row_0/B:col10/1733339744246/Put/seqid=0 2024-12-04T19:15:45,146 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241204efd850fb5f4b4e2787762602d0ab2958_77154c9b50d6b1163c1785212a1cb410 store=[table=TestAcidGuarantees family=A region=77154c9b50d6b1163c1785212a1cb410] 2024-12-04T19:15:45,147 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241204efd850fb5f4b4e2787762602d0ab2958_77154c9b50d6b1163c1785212a1cb410, store=[table=TestAcidGuarantees family=A region=77154c9b50d6b1163c1785212a1cb410] 2024-12-04T19:15:45,147 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204efd850fb5f4b4e2787762602d0ab2958_77154c9b50d6b1163c1785212a1cb410 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=77154c9b50d6b1163c1785212a1cb410] 2024-12-04T19:15:45,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742258_1434 (size=13085) 2024-12-04T19:15:45,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742259_1435 (size=4469) 2024-12-04T19:15:45,168 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-12-04T19:15:45,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:45,168 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing 77154c9b50d6b1163c1785212a1cb410 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-04T19:15:45,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77154c9b50d6b1163c1785212a1cb410, store=A 2024-12-04T19:15:45,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:15:45,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77154c9b50d6b1163c1785212a1cb410, store=B 2024-12-04T19:15:45,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:15:45,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77154c9b50d6b1163c1785212a1cb410, store=C 2024-12-04T19:15:45,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:15:45,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204fb41753a6d15429a8cc48476b30577c7_77154c9b50d6b1163c1785212a1cb410 is 50, key is test_row_0/A:col10/1733339744271/Put/seqid=0 2024-12-04T19:15:45,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742260_1436 (size=12454) 2024-12-04T19:15:45,186 INFO [Thread-1562 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1562. 2024-12-04T19:15:45,186 DEBUG [Thread-1562 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T19:15:45,186 DEBUG [Thread-1562 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:15:45,186 DEBUG [Thread-1562 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:15:45,186 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T19:15:45,191 INFO [Thread-1564 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1564. 2024-12-04T19:15:45,191 DEBUG [Thread-1564 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T19:15:45,191 DEBUG [Thread-1564 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:15:45,192 DEBUG [Thread-1564 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:15:45,192 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T19:15:45,192 INFO [Thread-1566 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1566. 2024-12-04T19:15:45,192 DEBUG [Thread-1566 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T19:15:45,192 DEBUG [Thread-1566 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:15:45,192 DEBUG [Thread-1566 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:15:45,192 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T19:15:45,194 INFO [Thread-1570 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1570. 2024-12-04T19:15:45,194 DEBUG [Thread-1570 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T19:15:45,194 DEBUG [Thread-1570 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:15:45,194 DEBUG [Thread-1570 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:15:45,194 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T19:15:45,197 INFO [Thread-1568 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1568. 2024-12-04T19:15:45,197 DEBUG [Thread-1568 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T19:15:45,197 DEBUG [Thread-1568 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:15:45,197 DEBUG [Thread-1568 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:15:45,197 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T19:15:45,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(8855): Flush requested on 77154c9b50d6b1163c1785212a1cb410 2024-12-04T19:15:45,433 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. as already flushing 2024-12-04T19:15:45,433 INFO [Thread-1555 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1555. 2024-12-04T19:15:45,433 INFO [Thread-1559 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1559. 2024-12-04T19:15:45,433 DEBUG [Thread-1555 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T19:15:45,433 INFO [Thread-1551 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1551. 2024-12-04T19:15:45,433 DEBUG [Thread-1559 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T19:15:45,433 DEBUG [Thread-1555 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:15:45,434 DEBUG [Thread-1559 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:15:45,434 DEBUG [Thread-1551 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T19:15:45,434 DEBUG [Thread-1551 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:15:45,434 DEBUG [Thread-1559 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:15:45,434 DEBUG [Thread-1555 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:15:45,434 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T19:15:45,434 DEBUG [Thread-1551 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:15:45,434 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T19:15:45,434 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T19:15:45,435 INFO [Thread-1557 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1557. 2024-12-04T19:15:45,435 DEBUG [Thread-1557 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T19:15:45,435 DEBUG [Thread-1557 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:15:45,435 DEBUG [Thread-1557 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:15:45,435 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T19:15:45,444 INFO [Thread-1553 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1553. 2024-12-04T19:15:45,444 DEBUG [Thread-1553 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T19:15:45,444 DEBUG [Thread-1553 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:15:45,444 DEBUG [Thread-1553 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:15:45,444 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T19:15:45,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-04T19:15:45,551 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/B/8927d5e142f341bf9090c41a5390cb59 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/8927d5e142f341bf9090c41a5390cb59 2024-12-04T19:15:45,554 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 77154c9b50d6b1163c1785212a1cb410#A#compaction#363 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:15:45,554 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/A/e8f80da404a9456fa0ac9ee132789fe2 is 175, key is test_row_0/A:col10/1733339744246/Put/seqid=0 2024-12-04T19:15:45,555 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 77154c9b50d6b1163c1785212a1cb410/B of 77154c9b50d6b1163c1785212a1cb410 into 8927d5e142f341bf9090c41a5390cb59(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:15:45,555 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 77154c9b50d6b1163c1785212a1cb410: 2024-12-04T19:15:45,555 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., storeName=77154c9b50d6b1163c1785212a1cb410/B, priority=13, startTime=1733339745131; duration=0sec 2024-12-04T19:15:45,555 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T19:15:45,555 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 77154c9b50d6b1163c1785212a1cb410:B 2024-12-04T19:15:45,555 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T19:15:45,556 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T19:15:45,556 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1541): 77154c9b50d6b1163c1785212a1cb410/C is initiating minor compaction (all files) 2024-12-04T19:15:45,556 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 77154c9b50d6b1163c1785212a1cb410/C in TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:45,556 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/2e41703bbaeb4fac81790d73df7094f8, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/aba22f12c89f4dd9900972e0d7b810c7, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/30cad4ba4bee4e5690d632e6008bf925] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp, totalSize=36.7 K 2024-12-04T19:15:45,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742261_1437 (size=32039) 2024-12-04T19:15:45,557 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e41703bbaeb4fac81790d73df7094f8, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1733339741265 2024-12-04T19:15:45,557 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting aba22f12c89f4dd9900972e0d7b810c7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1733339741923 2024-12-04T19:15:45,557 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 30cad4ba4bee4e5690d632e6008bf925, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1733339744131 2024-12-04T19:15:45,563 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 77154c9b50d6b1163c1785212a1cb410#C#compaction#365 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:15:45,563 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/C/c9264b08e91d4c0fa80248d3295bdf87 is 50, key is test_row_0/C:col10/1733339744246/Put/seqid=0 2024-12-04T19:15:45,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742262_1438 (size=13085) 2024-12-04T19:15:45,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:45,590 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204fb41753a6d15429a8cc48476b30577c7_77154c9b50d6b1163c1785212a1cb410 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204fb41753a6d15429a8cc48476b30577c7_77154c9b50d6b1163c1785212a1cb410 2024-12-04T19:15:45,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/A/a9aca20acf064fb9811769bfa89b8ae4, store: [table=TestAcidGuarantees family=A region=77154c9b50d6b1163c1785212a1cb410] 2024-12-04T19:15:45,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/A/a9aca20acf064fb9811769bfa89b8ae4 is 175, key is test_row_0/A:col10/1733339744271/Put/seqid=0 2024-12-04T19:15:45,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742263_1439 (size=31255) 2024-12-04T19:15:45,961 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/A/e8f80da404a9456fa0ac9ee132789fe2 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/e8f80da404a9456fa0ac9ee132789fe2 2024-12-04T19:15:45,965 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 77154c9b50d6b1163c1785212a1cb410/A of 77154c9b50d6b1163c1785212a1cb410 into e8f80da404a9456fa0ac9ee132789fe2(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:15:45,965 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 77154c9b50d6b1163c1785212a1cb410: 2024-12-04T19:15:45,965 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., storeName=77154c9b50d6b1163c1785212a1cb410/A, priority=13, startTime=1733339745130; duration=0sec 2024-12-04T19:15:45,966 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:15:45,966 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 77154c9b50d6b1163c1785212a1cb410:A 2024-12-04T19:15:45,971 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/C/c9264b08e91d4c0fa80248d3295bdf87 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/c9264b08e91d4c0fa80248d3295bdf87 2024-12-04T19:15:45,974 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 77154c9b50d6b1163c1785212a1cb410/C of 77154c9b50d6b1163c1785212a1cb410 into c9264b08e91d4c0fa80248d3295bdf87(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:15:45,974 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 77154c9b50d6b1163c1785212a1cb410: 2024-12-04T19:15:45,974 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., storeName=77154c9b50d6b1163c1785212a1cb410/C, priority=13, startTime=1733339745134; duration=0sec 2024-12-04T19:15:45,974 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:15:45,974 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 77154c9b50d6b1163c1785212a1cb410:C 2024-12-04T19:15:45,995 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=401, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/A/a9aca20acf064fb9811769bfa89b8ae4 2024-12-04T19:15:46,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/B/b71827ea737c48eab16f536e03a84376 is 50, key is test_row_0/B:col10/1733339744271/Put/seqid=0 2024-12-04T19:15:46,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742264_1440 (size=12301) 2024-12-04T19:15:46,406 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=401 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/B/b71827ea737c48eab16f536e03a84376 2024-12-04T19:15:46,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/C/772e926cab5e4f7fa7fc082ad3061614 is 50, key is test_row_0/C:col10/1733339744271/Put/seqid=0 2024-12-04T19:15:46,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742265_1441 (size=12301) 2024-12-04T19:15:46,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-04T19:15:46,821 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=401 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/C/772e926cab5e4f7fa7fc082ad3061614 2024-12-04T19:15:46,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/A/a9aca20acf064fb9811769bfa89b8ae4 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/a9aca20acf064fb9811769bfa89b8ae4 2024-12-04T19:15:46,830 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/a9aca20acf064fb9811769bfa89b8ae4, entries=150, sequenceid=401, filesize=30.5 K 2024-12-04T19:15:46,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/B/b71827ea737c48eab16f536e03a84376 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/b71827ea737c48eab16f536e03a84376 2024-12-04T19:15:46,834 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/b71827ea737c48eab16f536e03a84376, entries=150, sequenceid=401, filesize=12.0 K 2024-12-04T19:15:46,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/C/772e926cab5e4f7fa7fc082ad3061614 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/772e926cab5e4f7fa7fc082ad3061614 2024-12-04T19:15:46,838 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/772e926cab5e4f7fa7fc082ad3061614, entries=150, sequenceid=401, filesize=12.0 K 2024-12-04T19:15:46,838 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=33.54 KB/34350 for 77154c9b50d6b1163c1785212a1cb410 in 1670ms, sequenceid=401, compaction requested=false 2024-12-04T19:15:46,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for 77154c9b50d6b1163c1785212a1cb410: 2024-12-04T19:15:46,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:46,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-12-04T19:15:46,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-12-04T19:15:46,841 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-12-04T19:15:46,841 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4360 sec 2024-12-04T19:15:46,842 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 2.4400 sec 2024-12-04T19:15:48,014 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-04T19:15:48,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-04T19:15:48,574 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T19:15:48,574 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-04T19:15:48,574 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 99 2024-12-04T19:15:48,574 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 46 2024-12-04T19:15:48,574 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 60 2024-12-04T19:15:48,574 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 46 2024-12-04T19:15:48,574 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 79 2024-12-04T19:15:48,574 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-04T19:15:48,574 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-04T19:15:48,574 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1445 2024-12-04T19:15:48,574 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4325 rows 2024-12-04T19:15:48,574 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1439 2024-12-04T19:15:48,574 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4313 rows 2024-12-04T19:15:48,574 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1435 2024-12-04T19:15:48,574 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4302 rows 2024-12-04T19:15:48,574 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1457 2024-12-04T19:15:48,574 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4368 rows 2024-12-04T19:15:48,574 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1441 2024-12-04T19:15:48,574 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4309 rows 2024-12-04T19:15:48,575 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-04T19:15:48,575 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T19:15:48,575 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:15:48,575 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:15:48,575 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T19:15:48,577 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-04T19:15:48,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-04T19:15:48,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-04T19:15:48,583 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733339748583"}]},"ts":"1733339748583"} 2024-12-04T19:15:48,584 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-04T19:15:48,585 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-04T19:15:48,585 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-04T19:15:48,587 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=77154c9b50d6b1163c1785212a1cb410, UNASSIGN}] 2024-12-04T19:15:48,587 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=77154c9b50d6b1163c1785212a1cb410, UNASSIGN 2024-12-04T19:15:48,588 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=115 updating hbase:meta row=77154c9b50d6b1163c1785212a1cb410, regionState=CLOSING, regionLocation=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:48,589 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=77154c9b50d6b1163c1785212a1cb410, UNASSIGN because future has completed 2024-12-04T19:15:48,590 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-04T19:15:48,590 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; CloseRegionProcedure 77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791}] 2024-12-04T19:15:48,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-04T19:15:48,743 INFO [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] handler.UnassignRegionHandler(122): Close 77154c9b50d6b1163c1785212a1cb410 2024-12-04T19:15:48,743 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-04T19:15:48,743 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1722): Closing 77154c9b50d6b1163c1785212a1cb410, disabling compactions & flushes 2024-12-04T19:15:48,743 INFO [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:48,743 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:48,743 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. after waiting 0 ms 2024-12-04T19:15:48,743 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:48,744 INFO [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(2902): Flushing 77154c9b50d6b1163c1785212a1cb410 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-04T19:15:48,744 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77154c9b50d6b1163c1785212a1cb410, store=A 2024-12-04T19:15:48,744 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:15:48,744 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77154c9b50d6b1163c1785212a1cb410, store=B 2024-12-04T19:15:48,744 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:15:48,744 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77154c9b50d6b1163c1785212a1cb410, store=C 2024-12-04T19:15:48,744 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:15:48,754 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412044c4a9b4b6d06453aba5aeb9257b8b687_77154c9b50d6b1163c1785212a1cb410 is 50, key is test_row_0/A:col10/1733339745432/Put/seqid=0 2024-12-04T19:15:48,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742266_1442 (size=12454) 2024-12-04T19:15:48,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-04T19:15:49,161 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:49,169 INFO [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412044c4a9b4b6d06453aba5aeb9257b8b687_77154c9b50d6b1163c1785212a1cb410 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412044c4a9b4b6d06453aba5aeb9257b8b687_77154c9b50d6b1163c1785212a1cb410 2024-12-04T19:15:49,171 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/A/e482f84d7c5748e1a21516a79fa788eb, store: [table=TestAcidGuarantees family=A region=77154c9b50d6b1163c1785212a1cb410] 2024-12-04T19:15:49,171 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/A/e482f84d7c5748e1a21516a79fa788eb is 175, key is test_row_0/A:col10/1733339745432/Put/seqid=0 2024-12-04T19:15:49,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742267_1443 (size=31255) 2024-12-04T19:15:49,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-04T19:15:49,577 INFO [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=412, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/A/e482f84d7c5748e1a21516a79fa788eb 2024-12-04T19:15:49,590 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/B/d9bbeffd4b8649c59d9865a85dba8401 is 50, key is test_row_0/B:col10/1733339745432/Put/seqid=0 2024-12-04T19:15:49,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742268_1444 (size=12301) 2024-12-04T19:15:49,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-04T19:15:49,994 INFO [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=412 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/B/d9bbeffd4b8649c59d9865a85dba8401 2024-12-04T19:15:50,006 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/C/593907bc41994723958aac39a7fc91a3 is 50, key is test_row_0/C:col10/1733339745432/Put/seqid=0 2024-12-04T19:15:50,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742269_1445 (size=12301) 2024-12-04T19:15:50,410 INFO [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=412 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/C/593907bc41994723958aac39a7fc91a3 2024-12-04T19:15:50,420 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/A/e482f84d7c5748e1a21516a79fa788eb as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/e482f84d7c5748e1a21516a79fa788eb 2024-12-04T19:15:50,427 INFO [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/e482f84d7c5748e1a21516a79fa788eb, entries=150, sequenceid=412, filesize=30.5 K 2024-12-04T19:15:50,428 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/B/d9bbeffd4b8649c59d9865a85dba8401 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/d9bbeffd4b8649c59d9865a85dba8401 2024-12-04T19:15:50,433 INFO [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/d9bbeffd4b8649c59d9865a85dba8401, entries=150, sequenceid=412, filesize=12.0 K 2024-12-04T19:15:50,434 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/.tmp/C/593907bc41994723958aac39a7fc91a3 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/593907bc41994723958aac39a7fc91a3 2024-12-04T19:15:50,438 INFO [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/593907bc41994723958aac39a7fc91a3, entries=150, sequenceid=412, filesize=12.0 K 2024-12-04T19:15:50,439 INFO [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 77154c9b50d6b1163c1785212a1cb410 in 1696ms, sequenceid=412, compaction requested=true 2024-12-04T19:15:50,439 DEBUG [StoreCloser-TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/461d009f76954422b02d34e1df1615ea, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/298ef5210f394fd6a0a38b1d86f97459, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/0048142ce36c482da6d2550c83845be6, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/1e335ddc0d804bb6bd12adcc4a85f1e7, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/bd06c8d06ec74bd5a779b9035c601b68, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/fc62085152a9490baf45161760aaad8f, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/36032633cf874735889f0f4a857f573c, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/1cea96d199fc422ebf161ecf2a2735c7, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/ecc20430f3924e45a04862bd2db529cc, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/fe6ee7719f9d464ea9ce565df1ae1cc3, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/c5aba57e26c04f77949f3f11f589fd1d, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/a0a90c211cf042ca90f023610fa6afe4, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/e4440c1322d1499eb83623c0912d11d2, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/a8cdae5c61ff4e4a9d9e0f910f680934, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/c146e91979b94794863602166a63a165, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/c3a14afb61e14403a87f41132358dc15, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/96534a0bf8934c4db00ed3731f23d9ae, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/0565b50961be46798e2cd4ac7defe135, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/64793b4a956c439d8a405934a862bf1a, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/9cfba6d3dcc144549339d816380a0627, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/4e9535e87756459390ed89ad1b12c629, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/0f4ae009ea9642618a592ba20133bed6, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/56ffe4ffafbd4d6bb708440dd589cc71] to archive 2024-12-04T19:15:50,440 DEBUG [StoreCloser-TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-04T19:15:50,442 DEBUG [StoreCloser-TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/461d009f76954422b02d34e1df1615ea to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/461d009f76954422b02d34e1df1615ea 2024-12-04T19:15:50,443 DEBUG [StoreCloser-TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/298ef5210f394fd6a0a38b1d86f97459 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/298ef5210f394fd6a0a38b1d86f97459 2024-12-04T19:15:50,444 DEBUG [StoreCloser-TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/0048142ce36c482da6d2550c83845be6 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/0048142ce36c482da6d2550c83845be6 2024-12-04T19:15:50,445 DEBUG [StoreCloser-TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/1e335ddc0d804bb6bd12adcc4a85f1e7 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/1e335ddc0d804bb6bd12adcc4a85f1e7 2024-12-04T19:15:50,446 DEBUG [StoreCloser-TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/bd06c8d06ec74bd5a779b9035c601b68 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/bd06c8d06ec74bd5a779b9035c601b68 2024-12-04T19:15:50,446 DEBUG [StoreCloser-TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/fc62085152a9490baf45161760aaad8f to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/fc62085152a9490baf45161760aaad8f 2024-12-04T19:15:50,447 DEBUG [StoreCloser-TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/36032633cf874735889f0f4a857f573c to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/36032633cf874735889f0f4a857f573c 2024-12-04T19:15:50,448 DEBUG [StoreCloser-TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/1cea96d199fc422ebf161ecf2a2735c7 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/1cea96d199fc422ebf161ecf2a2735c7 2024-12-04T19:15:50,449 DEBUG [StoreCloser-TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/ecc20430f3924e45a04862bd2db529cc to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/ecc20430f3924e45a04862bd2db529cc 2024-12-04T19:15:50,449 DEBUG [StoreCloser-TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/fe6ee7719f9d464ea9ce565df1ae1cc3 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/fe6ee7719f9d464ea9ce565df1ae1cc3 2024-12-04T19:15:50,450 DEBUG [StoreCloser-TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/c5aba57e26c04f77949f3f11f589fd1d to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/c5aba57e26c04f77949f3f11f589fd1d 2024-12-04T19:15:50,450 DEBUG [StoreCloser-TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/a0a90c211cf042ca90f023610fa6afe4 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/a0a90c211cf042ca90f023610fa6afe4 2024-12-04T19:15:50,451 DEBUG [StoreCloser-TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/e4440c1322d1499eb83623c0912d11d2 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/e4440c1322d1499eb83623c0912d11d2 2024-12-04T19:15:50,452 DEBUG [StoreCloser-TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/a8cdae5c61ff4e4a9d9e0f910f680934 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/a8cdae5c61ff4e4a9d9e0f910f680934 2024-12-04T19:15:50,453 DEBUG [StoreCloser-TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/c146e91979b94794863602166a63a165 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/c146e91979b94794863602166a63a165 2024-12-04T19:15:50,453 DEBUG [StoreCloser-TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/c3a14afb61e14403a87f41132358dc15 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/c3a14afb61e14403a87f41132358dc15 2024-12-04T19:15:50,454 DEBUG [StoreCloser-TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/96534a0bf8934c4db00ed3731f23d9ae to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/96534a0bf8934c4db00ed3731f23d9ae 2024-12-04T19:15:50,455 DEBUG [StoreCloser-TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/0565b50961be46798e2cd4ac7defe135 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/0565b50961be46798e2cd4ac7defe135 2024-12-04T19:15:50,455 DEBUG [StoreCloser-TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/64793b4a956c439d8a405934a862bf1a to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/64793b4a956c439d8a405934a862bf1a 2024-12-04T19:15:50,456 DEBUG [StoreCloser-TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/9cfba6d3dcc144549339d816380a0627 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/9cfba6d3dcc144549339d816380a0627 2024-12-04T19:15:50,457 DEBUG [StoreCloser-TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/4e9535e87756459390ed89ad1b12c629 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/4e9535e87756459390ed89ad1b12c629 2024-12-04T19:15:50,457 DEBUG [StoreCloser-TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/0f4ae009ea9642618a592ba20133bed6 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/0f4ae009ea9642618a592ba20133bed6 2024-12-04T19:15:50,458 DEBUG [StoreCloser-TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/56ffe4ffafbd4d6bb708440dd589cc71 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/56ffe4ffafbd4d6bb708440dd589cc71 2024-12-04T19:15:50,459 DEBUG [StoreCloser-TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/32ba7b3fa95d42cc9206ac7c1c76ffea, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/a3424de3a58d438d8828ae52e9bae33a, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/920a4f50be99442191f5c925fec89136, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/6540f00e49b840bda8e16478daac2a28, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/091471b12ef44197b69e5d1e592c805d, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/95edfde51af24d9187456293eff41895, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/40b43faf0c7540bb9493721e67ca558b, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/49d869b5512f435c947da2b682ded9ec, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/60166ae1812f4567a6a9f7542e53a72e, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/719c10a50b5849648dcf5a458077078b, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/53d275afa3854625930ac048a95d17c3, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/155d7d4979744abc9ca6e57275404470, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/2341f190f80c4cfe8b91290e712cff4e, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/2453fe6dc3a74561a50ad0af81d751d7, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/5a37e6185dcd49cdb33cc429184bb0d0, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/101a15c649fe4c78bec3034716fa09c6, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/2f27d226c58947a08b77a9af70bd66bc, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/57ee717f0af74c269ae2bf715dccdf29, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/91617287c87249dea7a002166a74bb53, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/69fdbbbfbdb04ebbb9980fd63bfca030, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/1bf31b6eac194d8893610f519304ed5c, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/dca1e5524cac49af9e38e5d6f6c7fdac, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/67e78ffae6a04829a138343d254aeb11] to archive 2024-12-04T19:15:50,460 DEBUG [StoreCloser-TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-04T19:15:50,461 DEBUG [StoreCloser-TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/32ba7b3fa95d42cc9206ac7c1c76ffea to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/32ba7b3fa95d42cc9206ac7c1c76ffea 2024-12-04T19:15:50,462 DEBUG [StoreCloser-TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/a3424de3a58d438d8828ae52e9bae33a to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/a3424de3a58d438d8828ae52e9bae33a 2024-12-04T19:15:50,462 DEBUG [StoreCloser-TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/920a4f50be99442191f5c925fec89136 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/920a4f50be99442191f5c925fec89136 2024-12-04T19:15:50,463 DEBUG [StoreCloser-TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/6540f00e49b840bda8e16478daac2a28 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/6540f00e49b840bda8e16478daac2a28 2024-12-04T19:15:50,464 DEBUG [StoreCloser-TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/091471b12ef44197b69e5d1e592c805d to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/091471b12ef44197b69e5d1e592c805d 2024-12-04T19:15:50,464 DEBUG [StoreCloser-TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/95edfde51af24d9187456293eff41895 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/95edfde51af24d9187456293eff41895 2024-12-04T19:15:50,465 DEBUG [StoreCloser-TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/40b43faf0c7540bb9493721e67ca558b to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/40b43faf0c7540bb9493721e67ca558b 2024-12-04T19:15:50,466 DEBUG [StoreCloser-TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/49d869b5512f435c947da2b682ded9ec to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/49d869b5512f435c947da2b682ded9ec 2024-12-04T19:15:50,466 DEBUG [StoreCloser-TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/60166ae1812f4567a6a9f7542e53a72e to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/60166ae1812f4567a6a9f7542e53a72e 2024-12-04T19:15:50,467 DEBUG [StoreCloser-TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/719c10a50b5849648dcf5a458077078b to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/719c10a50b5849648dcf5a458077078b 2024-12-04T19:15:50,467 DEBUG [StoreCloser-TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/53d275afa3854625930ac048a95d17c3 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/53d275afa3854625930ac048a95d17c3 2024-12-04T19:15:50,468 DEBUG [StoreCloser-TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/155d7d4979744abc9ca6e57275404470 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/155d7d4979744abc9ca6e57275404470 2024-12-04T19:15:50,469 DEBUG [StoreCloser-TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/2341f190f80c4cfe8b91290e712cff4e to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/2341f190f80c4cfe8b91290e712cff4e 2024-12-04T19:15:50,469 DEBUG [StoreCloser-TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/2453fe6dc3a74561a50ad0af81d751d7 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/2453fe6dc3a74561a50ad0af81d751d7 2024-12-04T19:15:50,470 DEBUG [StoreCloser-TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/5a37e6185dcd49cdb33cc429184bb0d0 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/5a37e6185dcd49cdb33cc429184bb0d0 2024-12-04T19:15:50,470 DEBUG [StoreCloser-TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/101a15c649fe4c78bec3034716fa09c6 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/101a15c649fe4c78bec3034716fa09c6 2024-12-04T19:15:50,471 DEBUG [StoreCloser-TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/2f27d226c58947a08b77a9af70bd66bc to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/2f27d226c58947a08b77a9af70bd66bc 2024-12-04T19:15:50,471 DEBUG [StoreCloser-TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/57ee717f0af74c269ae2bf715dccdf29 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/57ee717f0af74c269ae2bf715dccdf29 2024-12-04T19:15:50,472 DEBUG [StoreCloser-TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/91617287c87249dea7a002166a74bb53 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/91617287c87249dea7a002166a74bb53 2024-12-04T19:15:50,473 DEBUG [StoreCloser-TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/69fdbbbfbdb04ebbb9980fd63bfca030 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/69fdbbbfbdb04ebbb9980fd63bfca030 2024-12-04T19:15:50,473 DEBUG [StoreCloser-TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/1bf31b6eac194d8893610f519304ed5c to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/1bf31b6eac194d8893610f519304ed5c 2024-12-04T19:15:50,474 DEBUG [StoreCloser-TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/dca1e5524cac49af9e38e5d6f6c7fdac to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/dca1e5524cac49af9e38e5d6f6c7fdac 2024-12-04T19:15:50,474 DEBUG [StoreCloser-TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/67e78ffae6a04829a138343d254aeb11 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/67e78ffae6a04829a138343d254aeb11 2024-12-04T19:15:50,475 DEBUG [StoreCloser-TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/6fa081a0b96449c9adf48594b89e1e5a, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/b07c0982f557424bafb470321f9902b7, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/891d348ee4974c1ebf10503be913cd89, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/72e48841d1804b4c8c543ff781208102, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/6db98868848b43cd9be3384962ea0010, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/29d5215236f54d418ed2a5a5985a9903, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/c7361166861f48bfbba7bbd4f9db6381, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/b9d34dd534aa40a38da0255b7ff2a80e, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/74cf0ea3af6d442fb4b0e070a3e01a20, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/7c00618b71b849f7b69461b5d688f95e, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/f81dee1a19334aa9a3508e6de4127930, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/a0fd7662bbf04f609067c68bd5911747, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/97fc60f723464a6988d46e9f5068fbde, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/90e2969d8b024e2ab765cfb235639356, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/d6ba68f6144d4ad99782223825f00e6f, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/80fe5af98dcc411abd20dd8d147a2b02, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/163ca0b18e8b46568837a97f7cd497e7, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/92657249449146fe9121a475d0295ae9, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/4896ec3b3b5a4c66a8321959377da7e4, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/2e41703bbaeb4fac81790d73df7094f8, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/269c7e8313b64f1fa03a05d8cd80be79, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/aba22f12c89f4dd9900972e0d7b810c7, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/30cad4ba4bee4e5690d632e6008bf925] to archive 2024-12-04T19:15:50,476 DEBUG [StoreCloser-TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-04T19:15:50,477 DEBUG [StoreCloser-TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/6fa081a0b96449c9adf48594b89e1e5a to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/6fa081a0b96449c9adf48594b89e1e5a 2024-12-04T19:15:50,478 DEBUG [StoreCloser-TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/b07c0982f557424bafb470321f9902b7 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/b07c0982f557424bafb470321f9902b7 2024-12-04T19:15:50,479 DEBUG [StoreCloser-TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/891d348ee4974c1ebf10503be913cd89 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/891d348ee4974c1ebf10503be913cd89 2024-12-04T19:15:50,480 DEBUG [StoreCloser-TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/72e48841d1804b4c8c543ff781208102 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/72e48841d1804b4c8c543ff781208102 2024-12-04T19:15:50,480 DEBUG [StoreCloser-TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/6db98868848b43cd9be3384962ea0010 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/6db98868848b43cd9be3384962ea0010 2024-12-04T19:15:50,481 DEBUG [StoreCloser-TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/29d5215236f54d418ed2a5a5985a9903 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/29d5215236f54d418ed2a5a5985a9903 2024-12-04T19:15:50,482 DEBUG [StoreCloser-TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/c7361166861f48bfbba7bbd4f9db6381 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/c7361166861f48bfbba7bbd4f9db6381 2024-12-04T19:15:50,482 DEBUG [StoreCloser-TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/b9d34dd534aa40a38da0255b7ff2a80e to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/b9d34dd534aa40a38da0255b7ff2a80e 2024-12-04T19:15:50,483 DEBUG [StoreCloser-TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/74cf0ea3af6d442fb4b0e070a3e01a20 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/74cf0ea3af6d442fb4b0e070a3e01a20 2024-12-04T19:15:50,484 DEBUG [StoreCloser-TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/7c00618b71b849f7b69461b5d688f95e to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/7c00618b71b849f7b69461b5d688f95e 2024-12-04T19:15:50,484 DEBUG [StoreCloser-TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/f81dee1a19334aa9a3508e6de4127930 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/f81dee1a19334aa9a3508e6de4127930 2024-12-04T19:15:50,485 DEBUG [StoreCloser-TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/a0fd7662bbf04f609067c68bd5911747 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/a0fd7662bbf04f609067c68bd5911747 2024-12-04T19:15:50,485 DEBUG [StoreCloser-TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/97fc60f723464a6988d46e9f5068fbde to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/97fc60f723464a6988d46e9f5068fbde 2024-12-04T19:15:50,486 DEBUG [StoreCloser-TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/90e2969d8b024e2ab765cfb235639356 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/90e2969d8b024e2ab765cfb235639356 2024-12-04T19:15:50,486 DEBUG [StoreCloser-TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/d6ba68f6144d4ad99782223825f00e6f to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/d6ba68f6144d4ad99782223825f00e6f 2024-12-04T19:15:50,487 DEBUG [StoreCloser-TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/80fe5af98dcc411abd20dd8d147a2b02 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/80fe5af98dcc411abd20dd8d147a2b02 2024-12-04T19:15:50,488 DEBUG [StoreCloser-TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/163ca0b18e8b46568837a97f7cd497e7 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/163ca0b18e8b46568837a97f7cd497e7 2024-12-04T19:15:50,488 DEBUG [StoreCloser-TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/92657249449146fe9121a475d0295ae9 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/92657249449146fe9121a475d0295ae9 2024-12-04T19:15:50,489 DEBUG [StoreCloser-TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/4896ec3b3b5a4c66a8321959377da7e4 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/4896ec3b3b5a4c66a8321959377da7e4 2024-12-04T19:15:50,490 DEBUG [StoreCloser-TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/2e41703bbaeb4fac81790d73df7094f8 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/2e41703bbaeb4fac81790d73df7094f8 2024-12-04T19:15:50,490 DEBUG [StoreCloser-TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/269c7e8313b64f1fa03a05d8cd80be79 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/269c7e8313b64f1fa03a05d8cd80be79 2024-12-04T19:15:50,491 DEBUG [StoreCloser-TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/aba22f12c89f4dd9900972e0d7b810c7 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/aba22f12c89f4dd9900972e0d7b810c7 2024-12-04T19:15:50,492 DEBUG [StoreCloser-TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/30cad4ba4bee4e5690d632e6008bf925 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/30cad4ba4bee4e5690d632e6008bf925 2024-12-04T19:15:50,495 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/recovered.edits/415.seqid, newMaxSeqId=415, maxSeqId=4 2024-12-04T19:15:50,495 INFO [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. 2024-12-04T19:15:50,495 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1676): Region close journal for 77154c9b50d6b1163c1785212a1cb410: Waiting for close lock at 1733339748743Running coprocessor pre-close hooks at 1733339748743Disabling compacts and flushes for region at 1733339748743Disabling writes for close at 1733339748743Obtaining lock to block concurrent updates at 1733339748744 (+1 ms)Preparing flush snapshotting stores in 77154c9b50d6b1163c1785212a1cb410 at 1733339748744Finished memstore snapshotting TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1733339748744Flushing stores of TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410. at 1733339748745 (+1 ms)Flushing 77154c9b50d6b1163c1785212a1cb410/A: creating writer at 1733339748745Flushing mob file 77154c9b50d6b1163c1785212a1cb410/A: appending metadata at 1733339748754 (+9 ms)Flushing mob file 77154c9b50d6b1163c1785212a1cb410/A: closing flushed file at 1733339748754Flushing 77154c9b50d6b1163c1785212a1cb410/A: appending metadata at 1733339749171 (+417 ms)Flushing 77154c9b50d6b1163c1785212a1cb410/A: closing flushed file at 1733339749171Flushing 77154c9b50d6b1163c1785212a1cb410/B: creating writer at 1733339749585 (+414 ms)Flushing 77154c9b50d6b1163c1785212a1cb410/B: appending metadata at 1733339749589 (+4 ms)Flushing 77154c9b50d6b1163c1785212a1cb410/B: closing flushed file at 1733339749589Flushing 77154c9b50d6b1163c1785212a1cb410/C: creating writer at 1733339750002 (+413 ms)Flushing 77154c9b50d6b1163c1785212a1cb410/C: appending metadata at 1733339750005 (+3 ms)Flushing 77154c9b50d6b1163c1785212a1cb410/C: closing flushed file at 1733339750005Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6519bd07: reopening flushed file at 1733339750419 (+414 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@12865ffa: reopening flushed file at 1733339750427 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@499472a9: reopening flushed file at 1733339750433 (+6 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 77154c9b50d6b1163c1785212a1cb410 in 1696ms, sequenceid=412, compaction requested=true at 1733339750439 (+6 ms)Writing region close event to WAL at 1733339750493 (+54 ms)Running coprocessor post-close hooks at 1733339750495 (+2 ms)Closed at 1733339750495 2024-12-04T19:15:50,497 INFO [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] handler.UnassignRegionHandler(157): Closed 77154c9b50d6b1163c1785212a1cb410 2024-12-04T19:15:50,497 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=115 updating hbase:meta row=77154c9b50d6b1163c1785212a1cb410, regionState=CLOSED 2024-12-04T19:15:50,498 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=116, ppid=115, state=RUNNABLE, hasLock=false; CloseRegionProcedure 77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 because future has completed 2024-12-04T19:15:50,499 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-12-04T19:15:50,500 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; CloseRegionProcedure 77154c9b50d6b1163c1785212a1cb410, server=9ba2734cd39f,33969,1733339629791 in 1.9080 sec 2024-12-04T19:15:50,501 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=115, resume processing ppid=114 2024-12-04T19:15:50,501 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, ppid=114, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=77154c9b50d6b1163c1785212a1cb410, UNASSIGN in 1.9130 sec 2024-12-04T19:15:50,502 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-12-04T19:15:50,502 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9160 sec 2024-12-04T19:15:50,503 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733339750503"}]},"ts":"1733339750503"} 2024-12-04T19:15:50,504 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-04T19:15:50,504 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-04T19:15:50,505 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9270 sec 2024-12-04T19:15:50,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-04T19:15:50,732 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-04T19:15:50,733 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-04T19:15:50,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T19:15:50,736 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=117, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T19:15:50,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-04T19:15:50,737 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=117, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T19:15:50,741 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410 2024-12-04T19:15:50,744 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A, FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B, FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C, FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/recovered.edits] 2024-12-04T19:15:50,747 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/a9aca20acf064fb9811769bfa89b8ae4 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/a9aca20acf064fb9811769bfa89b8ae4 2024-12-04T19:15:50,748 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/e482f84d7c5748e1a21516a79fa788eb to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/e482f84d7c5748e1a21516a79fa788eb 2024-12-04T19:15:50,749 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/e8f80da404a9456fa0ac9ee132789fe2 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/A/e8f80da404a9456fa0ac9ee132789fe2 2024-12-04T19:15:50,752 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/8927d5e142f341bf9090c41a5390cb59 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/8927d5e142f341bf9090c41a5390cb59 2024-12-04T19:15:50,753 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/b71827ea737c48eab16f536e03a84376 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/b71827ea737c48eab16f536e03a84376 2024-12-04T19:15:50,754 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/d9bbeffd4b8649c59d9865a85dba8401 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/B/d9bbeffd4b8649c59d9865a85dba8401 2024-12-04T19:15:50,756 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/593907bc41994723958aac39a7fc91a3 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/593907bc41994723958aac39a7fc91a3 2024-12-04T19:15:50,757 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/772e926cab5e4f7fa7fc082ad3061614 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/772e926cab5e4f7fa7fc082ad3061614 2024-12-04T19:15:50,759 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/c9264b08e91d4c0fa80248d3295bdf87 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/C/c9264b08e91d4c0fa80248d3295bdf87 2024-12-04T19:15:50,761 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/recovered.edits/415.seqid to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410/recovered.edits/415.seqid 2024-12-04T19:15:50,762 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/77154c9b50d6b1163c1785212a1cb410 2024-12-04T19:15:50,762 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-04T19:15:50,762 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-04T19:15:50,763 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-04T19:15:50,767 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412040e8ecdf10e794d948498c8c6ab0e5e57_77154c9b50d6b1163c1785212a1cb410 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412040e8ecdf10e794d948498c8c6ab0e5e57_77154c9b50d6b1163c1785212a1cb410 2024-12-04T19:15:50,768 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412041158fa1ef4dd47fc8f0e71bceedc52f8_77154c9b50d6b1163c1785212a1cb410 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412041158fa1ef4dd47fc8f0e71bceedc52f8_77154c9b50d6b1163c1785212a1cb410 2024-12-04T19:15:50,770 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120425cff7db38a440999ae77139441fdb88_77154c9b50d6b1163c1785212a1cb410 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120425cff7db38a440999ae77139441fdb88_77154c9b50d6b1163c1785212a1cb410 2024-12-04T19:15:50,772 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412042750cc672a3745c4a0c78e848f38ad1b_77154c9b50d6b1163c1785212a1cb410 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412042750cc672a3745c4a0c78e848f38ad1b_77154c9b50d6b1163c1785212a1cb410 2024-12-04T19:15:50,773 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120427eb51b7d2d94a5198d31163db18f57c_77154c9b50d6b1163c1785212a1cb410 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120427eb51b7d2d94a5198d31163db18f57c_77154c9b50d6b1163c1785212a1cb410 2024-12-04T19:15:50,774 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412043f9fd9b27f9e4049ad6e219d5a137ef8_77154c9b50d6b1163c1785212a1cb410 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412043f9fd9b27f9e4049ad6e219d5a137ef8_77154c9b50d6b1163c1785212a1cb410 2024-12-04T19:15:50,774 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412044c4a9b4b6d06453aba5aeb9257b8b687_77154c9b50d6b1163c1785212a1cb410 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412044c4a9b4b6d06453aba5aeb9257b8b687_77154c9b50d6b1163c1785212a1cb410 2024-12-04T19:15:50,775 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412045c44cb2ad8d8486885df4e118d9f6e2f_77154c9b50d6b1163c1785212a1cb410 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412045c44cb2ad8d8486885df4e118d9f6e2f_77154c9b50d6b1163c1785212a1cb410 2024-12-04T19:15:50,776 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412048854e24879fa4e9bb80ed65ebca695a5_77154c9b50d6b1163c1785212a1cb410 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412048854e24879fa4e9bb80ed65ebca695a5_77154c9b50d6b1163c1785212a1cb410 2024-12-04T19:15:50,777 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120489e62b609c6b471da28d2695a807095a_77154c9b50d6b1163c1785212a1cb410 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120489e62b609c6b471da28d2695a807095a_77154c9b50d6b1163c1785212a1cb410 2024-12-04T19:15:50,778 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412049c51e099a81d402583d9e517809dc47b_77154c9b50d6b1163c1785212a1cb410 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412049c51e099a81d402583d9e517809dc47b_77154c9b50d6b1163c1785212a1cb410 2024-12-04T19:15:50,779 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204a305a3e14f7e4b97a4e5d2171a6ee97f_77154c9b50d6b1163c1785212a1cb410 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204a305a3e14f7e4b97a4e5d2171a6ee97f_77154c9b50d6b1163c1785212a1cb410 2024-12-04T19:15:50,779 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204a8ac695a70424090b8078a5ef5dc1843_77154c9b50d6b1163c1785212a1cb410 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204a8ac695a70424090b8078a5ef5dc1843_77154c9b50d6b1163c1785212a1cb410 2024-12-04T19:15:50,780 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204c375f3c48af645b89335f07ec40ec475_77154c9b50d6b1163c1785212a1cb410 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204c375f3c48af645b89335f07ec40ec475_77154c9b50d6b1163c1785212a1cb410 2024-12-04T19:15:50,781 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204dc56acab3dd741beac96aae10318cd59_77154c9b50d6b1163c1785212a1cb410 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204dc56acab3dd741beac96aae10318cd59_77154c9b50d6b1163c1785212a1cb410 2024-12-04T19:15:50,782 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204f16e4649d97e4974a5541f29c76eb619_77154c9b50d6b1163c1785212a1cb410 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204f16e4649d97e4974a5541f29c76eb619_77154c9b50d6b1163c1785212a1cb410 2024-12-04T19:15:50,783 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204f1bdf7696d704a74a01801cdc989e896_77154c9b50d6b1163c1785212a1cb410 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204f1bdf7696d704a74a01801cdc989e896_77154c9b50d6b1163c1785212a1cb410 2024-12-04T19:15:50,784 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204fb41753a6d15429a8cc48476b30577c7_77154c9b50d6b1163c1785212a1cb410 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204fb41753a6d15429a8cc48476b30577c7_77154c9b50d6b1163c1785212a1cb410 2024-12-04T19:15:50,784 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204ff89b53e06d84cd7add79a2fe0f07f56_77154c9b50d6b1163c1785212a1cb410 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204ff89b53e06d84cd7add79a2fe0f07f56_77154c9b50d6b1163c1785212a1cb410 2024-12-04T19:15:50,785 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-04T19:15:50,787 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=117, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T19:15:50,789 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-04T19:15:50,791 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-04T19:15:50,792 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=117, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T19:15:50,792 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-04T19:15:50,793 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733339750792"}]},"ts":"9223372036854775807"} 2024-12-04T19:15:50,795 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-04T19:15:50,795 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 77154c9b50d6b1163c1785212a1cb410, NAME => 'TestAcidGuarantees,,1733339722035.77154c9b50d6b1163c1785212a1cb410.', STARTKEY => '', ENDKEY => ''}] 2024-12-04T19:15:50,795 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-04T19:15:50,795 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733339750795"}]},"ts":"9223372036854775807"} 2024-12-04T19:15:50,797 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-04T19:15:50,797 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=117, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T19:15:50,798 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 64 msec 2024-12-04T19:15:50,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-04T19:15:50,842 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-04T19:15:50,842 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-04T19:15:50,852 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=251 (was 250) - Thread LEAK? -, OpenFileDescriptor=461 (was 453) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=440 (was 456), ProcessCount=11 (was 11), AvailableMemoryMB=15613 (was 14920) - AvailableMemoryMB LEAK? - 2024-12-04T19:15:50,862 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=251, OpenFileDescriptor=461, MaxFileDescriptor=1048576, SystemLoadAverage=440, ProcessCount=11, AvailableMemoryMB=15613 2024-12-04T19:15:50,863 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-04T19:15:50,863 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T19:15:50,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] procedure2.ProcedureExecutor(1139): Stored pid=118, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-04T19:15:50,865 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-04T19:15:50,865 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:50,865 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 118 2024-12-04T19:15:50,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-12-04T19:15:50,866 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-04T19:15:50,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742270_1446 (size=1061) 2024-12-04T19:15:50,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-12-04T19:15:51,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-12-04T19:15:51,276 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d 2024-12-04T19:15:51,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742271_1447 (size=53) 2024-12-04T19:15:51,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-12-04T19:15:51,685 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T19:15:51,685 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 48c104a4c7a51d8b62d084d2a093043a, disabling compactions & flushes 2024-12-04T19:15:51,685 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:15:51,685 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:15:51,685 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. after waiting 0 ms 2024-12-04T19:15:51,685 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:15:51,685 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:15:51,685 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 48c104a4c7a51d8b62d084d2a093043a: Waiting for close lock at 1733339751685Disabling compacts and flushes for region at 1733339751685Disabling writes for close at 1733339751685Writing region close event to WAL at 1733339751685Closed at 1733339751685 2024-12-04T19:15:51,686 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-04T19:15:51,687 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733339751686"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733339751686"}]},"ts":"1733339751686"} 2024-12-04T19:15:51,688 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-04T19:15:51,689 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-04T19:15:51,689 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733339751689"}]},"ts":"1733339751689"} 2024-12-04T19:15:51,690 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-04T19:15:51,690 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=48c104a4c7a51d8b62d084d2a093043a, ASSIGN}] 2024-12-04T19:15:51,691 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=48c104a4c7a51d8b62d084d2a093043a, ASSIGN 2024-12-04T19:15:51,692 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=48c104a4c7a51d8b62d084d2a093043a, ASSIGN; state=OFFLINE, location=9ba2734cd39f,33969,1733339629791; forceNewPlan=false, retain=false 2024-12-04T19:15:51,843 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=119 updating hbase:meta row=48c104a4c7a51d8b62d084d2a093043a, regionState=OPENING, regionLocation=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:51,844 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=48c104a4c7a51d8b62d084d2a093043a, ASSIGN because future has completed 2024-12-04T19:15:51,844 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; OpenRegionProcedure 48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791}] 2024-12-04T19:15:51,999 INFO [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:15:51,999 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(7752): Opening region: {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} 2024-12-04T19:15:52,000 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 48c104a4c7a51d8b62d084d2a093043a 2024-12-04T19:15:52,000 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T19:15:52,000 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(7794): checking encryption for 48c104a4c7a51d8b62d084d2a093043a 2024-12-04T19:15:52,000 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(7797): checking classloading for 48c104a4c7a51d8b62d084d2a093043a 2024-12-04T19:15:52,001 INFO [StoreOpener-48c104a4c7a51d8b62d084d2a093043a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 48c104a4c7a51d8b62d084d2a093043a 2024-12-04T19:15:52,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-12-04T19:15:52,002 INFO [StoreOpener-48c104a4c7a51d8b62d084d2a093043a-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-04T19:15:52,003 INFO [StoreOpener-48c104a4c7a51d8b62d084d2a093043a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 48c104a4c7a51d8b62d084d2a093043a columnFamilyName A 2024-12-04T19:15:52,003 DEBUG [StoreOpener-48c104a4c7a51d8b62d084d2a093043a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:52,003 INFO [StoreOpener-48c104a4c7a51d8b62d084d2a093043a-1 {}] regionserver.HStore(327): Store=48c104a4c7a51d8b62d084d2a093043a/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:15:52,003 INFO [StoreOpener-48c104a4c7a51d8b62d084d2a093043a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 48c104a4c7a51d8b62d084d2a093043a 2024-12-04T19:15:52,004 INFO [StoreOpener-48c104a4c7a51d8b62d084d2a093043a-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-04T19:15:52,005 INFO [StoreOpener-48c104a4c7a51d8b62d084d2a093043a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 48c104a4c7a51d8b62d084d2a093043a columnFamilyName B 2024-12-04T19:15:52,005 DEBUG [StoreOpener-48c104a4c7a51d8b62d084d2a093043a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:52,005 INFO [StoreOpener-48c104a4c7a51d8b62d084d2a093043a-1 {}] regionserver.HStore(327): Store=48c104a4c7a51d8b62d084d2a093043a/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:15:52,005 INFO [StoreOpener-48c104a4c7a51d8b62d084d2a093043a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 48c104a4c7a51d8b62d084d2a093043a 2024-12-04T19:15:52,006 INFO [StoreOpener-48c104a4c7a51d8b62d084d2a093043a-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-04T19:15:52,006 INFO [StoreOpener-48c104a4c7a51d8b62d084d2a093043a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 48c104a4c7a51d8b62d084d2a093043a columnFamilyName C 2024-12-04T19:15:52,006 DEBUG [StoreOpener-48c104a4c7a51d8b62d084d2a093043a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:15:52,006 INFO [StoreOpener-48c104a4c7a51d8b62d084d2a093043a-1 {}] regionserver.HStore(327): Store=48c104a4c7a51d8b62d084d2a093043a/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:15:52,007 INFO [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:15:52,007 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1038): replaying wal for 48c104a4c7a51d8b62d084d2a093043a 2024-12-04T19:15:52,012 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a 2024-12-04T19:15:52,012 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a 2024-12-04T19:15:52,013 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1048): stopping wal replay for 48c104a4c7a51d8b62d084d2a093043a 2024-12-04T19:15:52,013 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1060): Cleaning up temporary data for 48c104a4c7a51d8b62d084d2a093043a 2024-12-04T19:15:52,013 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-04T19:15:52,014 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1093): writing seq id for 48c104a4c7a51d8b62d084d2a093043a 2024-12-04T19:15:52,015 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T19:15:52,016 INFO [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1114): Opened 48c104a4c7a51d8b62d084d2a093043a; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58830937, jitterRate=-0.12335072457790375}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-04T19:15:52,016 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 48c104a4c7a51d8b62d084d2a093043a 2024-12-04T19:15:52,016 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1006): Region open journal for 48c104a4c7a51d8b62d084d2a093043a: Running coprocessor pre-open hook at 1733339752000Writing region info on filesystem at 1733339752000Initializing all the Stores at 1733339752000Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733339752001 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733339752001Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733339752001Cleaning up temporary data from old regions at 1733339752013 (+12 ms)Running coprocessor post-open hooks at 1733339752016 (+3 ms)Region opened successfully at 1733339752016 2024-12-04T19:15:52,017 INFO [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., pid=120, masterSystemTime=1733339751996 2024-12-04T19:15:52,019 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:15:52,019 INFO [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:15:52,019 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=119 updating hbase:meta row=48c104a4c7a51d8b62d084d2a093043a, regionState=OPEN, openSeqNum=2, regionLocation=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:52,020 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=120, ppid=119, state=RUNNABLE, hasLock=false; OpenRegionProcedure 48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 because future has completed 2024-12-04T19:15:52,023 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-12-04T19:15:52,023 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; OpenRegionProcedure 48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 in 177 msec 2024-12-04T19:15:52,024 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=119, resume processing ppid=118 2024-12-04T19:15:52,024 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, ppid=118, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=48c104a4c7a51d8b62d084d2a093043a, ASSIGN in 333 msec 2024-12-04T19:15:52,025 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-04T19:15:52,025 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733339752025"}]},"ts":"1733339752025"} 2024-12-04T19:15:52,026 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-04T19:15:52,027 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-04T19:15:52,028 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1640 sec 2024-12-04T19:15:52,351 WARN [regionserver/9ba2734cd39f:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 6, running: 1 2024-12-04T19:15:53,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-12-04T19:15:53,013 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-04T19:15:53,014 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27239e80, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:15:53,014 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ba2734cd39f,46667,-1 for getting cluster id 2024-12-04T19:15:53,014 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T19:15:53,016 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9baaea35-8dd3-4653-9b7c-03e5b792e644' 2024-12-04T19:15:53,016 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T19:15:53,016 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9baaea35-8dd3-4653-9b7c-03e5b792e644" 2024-12-04T19:15:53,016 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49a4f96d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:15:53,016 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ba2734cd39f,46667,-1] 2024-12-04T19:15:53,017 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T19:15:53,017 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:15:53,019 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45698, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T19:15:53,020 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4df2c421, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:15:53,021 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T19:15:53,022 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ba2734cd39f,33969,1733339629791, seqNum=-1] 2024-12-04T19:15:53,022 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T19:15:53,024 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52492, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T19:15:53,025 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-04T19:15:53,026 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 9ba2734cd39f,46667,1733339628877 2024-12-04T19:15:53,026 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@39aeab91 2024-12-04T19:15:53,026 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-04T19:15:53,027 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45700, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-04T19:15:53,028 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@168cafba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:15:53,028 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ba2734cd39f,46667,-1 for getting cluster id 2024-12-04T19:15:53,029 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T19:15:53,029 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9baaea35-8dd3-4653-9b7c-03e5b792e644' 2024-12-04T19:15:53,029 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T19:15:53,029 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9baaea35-8dd3-4653-9b7c-03e5b792e644" 2024-12-04T19:15:53,029 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d7704c5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:15:53,029 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ba2734cd39f,46667,-1] 2024-12-04T19:15:53,030 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T19:15:53,030 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:15:53,030 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45706, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T19:15:53,031 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c3448ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:15:53,031 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d395438, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:15:53,031 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ba2734cd39f,46667,-1 for getting cluster id 2024-12-04T19:15:53,031 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T19:15:53,032 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9baaea35-8dd3-4653-9b7c-03e5b792e644' 2024-12-04T19:15:53,032 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T19:15:53,032 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9baaea35-8dd3-4653-9b7c-03e5b792e644" 2024-12-04T19:15:53,032 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6619c559, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:15:53,032 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ba2734cd39f,46667,-1] 2024-12-04T19:15:53,032 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T19:15:53,032 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:15:53,033 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45734, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T19:15:53,033 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b054b84, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:15:53,034 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b2d6985, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:15:53,034 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ba2734cd39f,46667,-1 for getting cluster id 2024-12-04T19:15:53,034 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T19:15:53,035 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9baaea35-8dd3-4653-9b7c-03e5b792e644' 2024-12-04T19:15:53,035 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T19:15:53,035 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9baaea35-8dd3-4653-9b7c-03e5b792e644" 2024-12-04T19:15:53,035 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@772665ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:15:53,035 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ba2734cd39f,46667,-1] 2024-12-04T19:15:53,035 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T19:15:53,035 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:15:53,036 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45750, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T19:15:53,036 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19bf58b4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:15:53,037 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@490627bb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:15:53,037 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ba2734cd39f,46667,-1 for getting cluster id 2024-12-04T19:15:53,037 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T19:15:53,037 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9baaea35-8dd3-4653-9b7c-03e5b792e644' 2024-12-04T19:15:53,037 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T19:15:53,038 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9baaea35-8dd3-4653-9b7c-03e5b792e644" 2024-12-04T19:15:53,038 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8120da2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:15:53,038 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ba2734cd39f,46667,-1] 2024-12-04T19:15:53,038 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T19:15:53,038 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:15:53,039 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45766, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T19:15:53,039 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68106fba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:15:53,040 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e16e9b9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:15:53,040 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ba2734cd39f,46667,-1 for getting cluster id 2024-12-04T19:15:53,040 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T19:15:53,040 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9baaea35-8dd3-4653-9b7c-03e5b792e644' 2024-12-04T19:15:53,040 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T19:15:53,040 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9baaea35-8dd3-4653-9b7c-03e5b792e644" 2024-12-04T19:15:53,041 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55d9b63b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:15:53,041 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ba2734cd39f,46667,-1] 2024-12-04T19:15:53,041 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T19:15:53,041 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:15:53,041 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45778, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T19:15:53,042 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b2d87ce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:15:53,042 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@794001fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:15:53,042 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ba2734cd39f,46667,-1 for getting cluster id 2024-12-04T19:15:53,042 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T19:15:53,043 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9baaea35-8dd3-4653-9b7c-03e5b792e644' 2024-12-04T19:15:53,043 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T19:15:53,043 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9baaea35-8dd3-4653-9b7c-03e5b792e644" 2024-12-04T19:15:53,043 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@515acd02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:15:53,043 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ba2734cd39f,46667,-1] 2024-12-04T19:15:53,043 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T19:15:53,043 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:15:53,044 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45792, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T19:15:53,044 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3721fffd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:15:53,045 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f3c4fbc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:15:53,045 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ba2734cd39f,46667,-1 for getting cluster id 2024-12-04T19:15:53,045 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T19:15:53,046 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9baaea35-8dd3-4653-9b7c-03e5b792e644' 2024-12-04T19:15:53,046 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T19:15:53,046 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9baaea35-8dd3-4653-9b7c-03e5b792e644" 2024-12-04T19:15:53,046 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@449f7430, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:15:53,046 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ba2734cd39f,46667,-1] 2024-12-04T19:15:53,046 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T19:15:53,046 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:15:53,047 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45818, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T19:15:53,047 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fe46e2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:15:53,048 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@732b24ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:15:53,048 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ba2734cd39f,46667,-1 for getting cluster id 2024-12-04T19:15:53,048 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T19:15:53,048 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9baaea35-8dd3-4653-9b7c-03e5b792e644' 2024-12-04T19:15:53,048 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T19:15:53,048 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9baaea35-8dd3-4653-9b7c-03e5b792e644" 2024-12-04T19:15:53,048 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20822bea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:15:53,048 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ba2734cd39f,46667,-1] 2024-12-04T19:15:53,049 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T19:15:53,049 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:15:53,049 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45834, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T19:15:53,050 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@733b1423, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:15:53,050 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1eaa8538, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:15:53,050 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ba2734cd39f,46667,-1 for getting cluster id 2024-12-04T19:15:53,050 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T19:15:53,051 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9baaea35-8dd3-4653-9b7c-03e5b792e644' 2024-12-04T19:15:53,051 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T19:15:53,051 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9baaea35-8dd3-4653-9b7c-03e5b792e644" 2024-12-04T19:15:53,051 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75762dd9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:15:53,051 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ba2734cd39f,46667,-1] 2024-12-04T19:15:53,051 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T19:15:53,051 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:15:53,052 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45856, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T19:15:53,052 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a2cc20f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:15:53,053 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67669ab8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:15:53,053 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ba2734cd39f,46667,-1 for getting cluster id 2024-12-04T19:15:53,053 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T19:15:53,053 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9baaea35-8dd3-4653-9b7c-03e5b792e644' 2024-12-04T19:15:53,053 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T19:15:53,053 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9baaea35-8dd3-4653-9b7c-03e5b792e644" 2024-12-04T19:15:53,053 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6643fd79, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:15:53,054 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ba2734cd39f,46667,-1] 2024-12-04T19:15:53,054 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T19:15:53,054 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:15:53,054 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45880, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T19:15:53,055 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54d89538, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:15:53,055 DEBUG [Thread-2018 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T19:15:53,055 DEBUG [Thread-2005 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T19:15:53,055 DEBUG [Thread-2009 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T19:15:53,055 DEBUG [Thread-2020 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T19:15:53,055 DEBUG [Thread-2014 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T19:15:53,056 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ba2734cd39f,33969,1733339629791, seqNum=-1] 2024-12-04T19:15:53,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ba2734cd39f,33969,1733339629791, seqNum=-1] 2024-12-04T19:15:53,056 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ba2734cd39f,33969,1733339629791, seqNum=-1] 2024-12-04T19:15:53,056 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ba2734cd39f,33969,1733339629791, seqNum=-1] 2024-12-04T19:15:53,056 DEBUG [Thread-2011 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T19:15:53,056 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T19:15:53,056 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ba2734cd39f,33969,1733339629791, seqNum=-1] 2024-12-04T19:15:53,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T19:15:53,056 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T19:15:53,056 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T19:15:53,056 DEBUG [Thread-2020 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T19:15:53,056 DEBUG [Thread-2003 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T19:15:53,057 DEBUG [Thread-2007 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T19:15:53,057 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ba2734cd39f,33969,1733339629791, seqNum=-1] 2024-12-04T19:15:53,057 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ba2734cd39f,33969,1733339629791, seqNum=-1] 2024-12-04T19:15:53,057 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T19:15:53,057 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T19:15:53,057 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52514, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T19:15:53,057 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52494, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T19:15:53,057 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T19:15:53,057 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52534, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T19:15:53,057 DEBUG [Thread-2016 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T19:15:53,058 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ba2734cd39f,33969,1733339629791, seqNum=-1] 2024-12-04T19:15:53,058 DEBUG [Thread-2022 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T19:15:53,058 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T19:15:53,058 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52496, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T19:15:53,058 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52544, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T19:15:53,058 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ba2734cd39f,33969,1733339629791, seqNum=-1] 2024-12-04T19:15:53,058 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52526, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T19:15:53,058 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T19:15:53,058 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ba2734cd39f,33969,1733339629791, seqNum=-1] 2024-12-04T19:15:53,059 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52546, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T19:15:53,059 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T19:15:53,059 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52562, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T19:15:53,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-12-04T19:15:53,059 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52576, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T19:15:53,060 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2] 2024-12-04T19:15:53,060 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52588, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T19:15:53,060 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T19:15:53,061 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2] 2024-12-04T19:15:53,061 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2] 2024-12-04T19:15:53,061 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2] 2024-12-04T19:15:53,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2] 2024-12-04T19:15:53,062 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2] 2024-12-04T19:15:53,062 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2] 2024-12-04T19:15:53,062 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2] 2024-12-04T19:15:53,064 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2] 2024-12-04T19:15:53,064 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T19:15:53,064 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T19:15:53,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-04T19:15:53,065 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2] 2024-12-04T19:15:53,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(8855): Flush requested on 48c104a4c7a51d8b62d084d2a093043a 2024-12-04T19:15:53,068 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 48c104a4c7a51d8b62d084d2a093043a 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-04T19:15:53,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 48c104a4c7a51d8b62d084d2a093043a, store=A 2024-12-04T19:15:53,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:15:53,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 48c104a4c7a51d8b62d084d2a093043a, store=B 2024-12-04T19:15:53,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:15:53,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 48c104a4c7a51d8b62d084d2a093043a, store=C 2024-12-04T19:15:53,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:15:53,094 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/A/9febb8cc21dd47f695ba56f16409c0aa is 50, key is test_row_0/A:col10/1733339753067/Put/seqid=0 2024-12-04T19:15:53,098 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:53,098 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:53,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52588 deadline: 1733339813090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:53,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52546 deadline: 1733339813092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:53,099 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:53,099 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:53,099 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:53,099 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:53,099 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:53,099 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:53,099 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733339753087, max timestamp=1733339753097Num uniques 859; 2024-12-04T19:15:53,099 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T19:15:53,099 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733339753087, max timestamp=1733339753097Num uniques 859; 2024-12-04T19:15:53,099 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T19:15:53,101 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733339753087, max timestamp=1733339753097Num uniques 859; 2024-12-04T19:15:53,101 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-04T19:15:53,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742272_1448 (size=16681) 2024-12-04T19:15:53,162 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:53,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52494 deadline: 1733339813152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:53,162 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:53,162 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:53,162 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:53,169 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:53,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52496 deadline: 1733339813160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:53,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:53,170 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:53,170 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:53,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52544 deadline: 1733339813162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:53,170 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:53,170 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:53,170 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:53,170 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:53,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-04T19:15:53,208 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:53,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52546 deadline: 1733339813202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:53,209 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:53,209 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:53,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52588 deadline: 1733339813202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:53,209 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:53,209 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:53,210 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:53,210 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:53,210 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:53,217 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-04T19:15:53,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:15:53,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. as already flushing 2024-12-04T19:15:53,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:15:53,217 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:53,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:53,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:53,274 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:53,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52494 deadline: 1733339813271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:53,275 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:53,275 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:53,275 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:53,275 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:53,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52496 deadline: 1733339813271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:53,275 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:53,275 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:53,275 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:53,285 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:53,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52544 deadline: 1733339813282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:53,285 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:53,286 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:53,286 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:53,370 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-04T19:15:53,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:15:53,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. as already flushing 2024-12-04T19:15:53,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:15:53,371 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:53,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:53,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:53,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-04T19:15:53,421 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:53,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52546 deadline: 1733339813411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:53,422 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:53,422 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:53,422 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:53,425 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:53,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52588 deadline: 1733339813422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:53,426 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:53,426 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:53,426 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:53,488 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:53,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52496 deadline: 1733339813482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:53,489 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:53,489 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:53,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52494 deadline: 1733339813482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:53,489 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:53,489 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:53,489 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:53,489 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:53,489 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:53,500 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:53,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52544 deadline: 1733339813492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:53,500 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:53,501 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:53,501 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:53,504 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/A/9febb8cc21dd47f695ba56f16409c0aa 2024-12-04T19:15:53,525 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-04T19:15:53,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:15:53,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. as already flushing 2024-12-04T19:15:53,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:15:53,525 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:53,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:53,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:53,530 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/B/2e1082f0347e42afad92ca5855a1e4ed is 50, key is test_row_0/B:col10/1733339753067/Put/seqid=0 2024-12-04T19:15:53,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742273_1449 (size=12001) 2024-12-04T19:15:53,677 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-04T19:15:53,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:15:53,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. as already flushing 2024-12-04T19:15:53,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:15:53,678 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:53,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:53,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:53,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-04T19:15:53,737 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:53,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52546 deadline: 1733339813732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:53,738 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:53,738 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:53,738 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:53,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52588 deadline: 1733339813732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:53,738 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:53,738 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:53,738 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:53,738 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:53,798 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:53,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52496 deadline: 1733339813792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:53,798 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:53,798 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:53,798 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:53,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52494 deadline: 1733339813792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:53,798 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:53,799 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:53,799 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:53,799 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:53,814 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:53,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52544 deadline: 1733339813811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:53,815 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:53,815 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:53,815 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:53,830 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-04T19:15:53,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:15:53,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. as already flushing 2024-12-04T19:15:53,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:15:53,830 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:53,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:53,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:53,934 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/B/2e1082f0347e42afad92ca5855a1e4ed 2024-12-04T19:15:53,958 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/C/6def83e5daaa470cad924c91e8427da0 is 50, key is test_row_0/C:col10/1733339753067/Put/seqid=0 2024-12-04T19:15:53,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742274_1450 (size=12001) 2024-12-04T19:15:53,983 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-04T19:15:53,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:15:53,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. as already flushing 2024-12-04T19:15:53,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:15:53,983 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:53,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:53,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:54,135 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-04T19:15:54,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:15:54,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. as already flushing 2024-12-04T19:15:54,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:15:54,136 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:54,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:54,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:54,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-04T19:15:54,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:54,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52546 deadline: 1733339814242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:54,248 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:54,248 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:54,248 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:54,248 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:54,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52588 deadline: 1733339814242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:54,248 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:54,248 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:54,248 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:54,288 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-04T19:15:54,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:15:54,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. as already flushing 2024-12-04T19:15:54,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:15:54,288 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:54,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:54,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:54,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:54,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52496 deadline: 1733339814312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:54,317 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:54,317 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:54,317 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:54,321 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:54,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52494 deadline: 1733339814313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:54,321 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:54,321 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:54,321 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:54,329 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:54,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52544 deadline: 1733339814321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:54,330 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:54,330 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:54,330 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:54,363 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/C/6def83e5daaa470cad924c91e8427da0 2024-12-04T19:15:54,366 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/A/9febb8cc21dd47f695ba56f16409c0aa as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/9febb8cc21dd47f695ba56f16409c0aa 2024-12-04T19:15:54,369 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/9febb8cc21dd47f695ba56f16409c0aa, entries=250, sequenceid=12, filesize=16.3 K 2024-12-04T19:15:54,370 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/B/2e1082f0347e42afad92ca5855a1e4ed as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/2e1082f0347e42afad92ca5855a1e4ed 2024-12-04T19:15:54,372 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/2e1082f0347e42afad92ca5855a1e4ed, entries=150, sequenceid=12, filesize=11.7 K 2024-12-04T19:15:54,373 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/C/6def83e5daaa470cad924c91e8427da0 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/6def83e5daaa470cad924c91e8427da0 2024-12-04T19:15:54,375 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/6def83e5daaa470cad924c91e8427da0, entries=150, sequenceid=12, filesize=11.7 K 2024-12-04T19:15:54,376 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.08 KB/130125 for 48c104a4c7a51d8b62d084d2a093043a in 1308ms, sequenceid=12, compaction requested=false 2024-12-04T19:15:54,376 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 48c104a4c7a51d8b62d084d2a093043a: 2024-12-04T19:15:54,441 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-04T19:15:54,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:15:54,441 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing 48c104a4c7a51d8b62d084d2a093043a 3/3 column families, dataSize=127.08 KB heapSize=364.15 KB 2024-12-04T19:15:54,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 48c104a4c7a51d8b62d084d2a093043a, store=A 2024-12-04T19:15:54,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T19:15:54,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 48c104a4c7a51d8b62d084d2a093043a, store=B 2024-12-04T19:15:54,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T19:15:54,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 48c104a4c7a51d8b62d084d2a093043a, store=C 2024-12-04T19:15:54,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T19:15:54,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/A/8909666709d74547b527b560d0e9903e is 50, key is test_row_0/A:col10/1733339753150/Put/seqid=0 2024-12-04T19:15:54,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742275_1451 (size=12001) 2024-12-04T19:15:54,851 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/A/8909666709d74547b527b560d0e9903e 2024-12-04T19:15:54,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/B/883ae9d5e2a84ce1a45c75da78312688 is 50, key is test_row_0/B:col10/1733339753150/Put/seqid=0 2024-12-04T19:15:54,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742276_1452 (size=12001) 2024-12-04T19:15:55,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-04T19:15:55,260 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/B/883ae9d5e2a84ce1a45c75da78312688 2024-12-04T19:15:55,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/C/069acd4b31004462a3bf5807487fcbce is 50, key is test_row_0/C:col10/1733339753150/Put/seqid=0 2024-12-04T19:15:55,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(8855): Flush requested on 48c104a4c7a51d8b62d084d2a093043a 2024-12-04T19:15:55,267 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. as already flushing 2024-12-04T19:15:55,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742277_1453 (size=12001) 2024-12-04T19:15:55,294 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:55,294 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:55,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52546 deadline: 1733339815282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:55,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52588 deadline: 1733339815286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:55,294 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:55,294 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:55,294 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:55,294 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:55,294 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:55,294 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:55,329 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:55,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52496 deadline: 1733339815322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:55,329 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 9ba2734cd39f,33969,1733339629791 for 'test_row_2' in 48c104a4c7a51d8b62d084d2a093043a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-04T19:15:55,330 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:55,330 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:55,330 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:55,338 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:55,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52494 deadline: 1733339815331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:55,338 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 9ba2734cd39f,33969,1733339629791 for 'test_row_1' in 48c104a4c7a51d8b62d084d2a093043a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-04T19:15:55,339 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:55,339 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:55,339 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:55,346 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:55,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52544 deadline: 1733339815341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:55,347 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 9ba2734cd39f,33969,1733339629791 for 'test_row_0' in 48c104a4c7a51d8b62d084d2a093043a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-04T19:15:55,347 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:55,347 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:55,347 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:55,405 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:55,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52588 deadline: 1733339815402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:55,405 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:55,405 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:55,405 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:55,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52546 deadline: 1733339815402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:55,405 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:55,405 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:55,406 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:55,406 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:55,618 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:55,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52588 deadline: 1733339815612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:55,618 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:55,618 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:55,618 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:55,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52546 deadline: 1733339815612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:55,618 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:55,618 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:55,618 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:55,618 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:55,671 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.09 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/C/069acd4b31004462a3bf5807487fcbce 2024-12-04T19:15:55,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/A/8909666709d74547b527b560d0e9903e as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/8909666709d74547b527b560d0e9903e 2024-12-04T19:15:55,677 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/8909666709d74547b527b560d0e9903e, entries=150, sequenceid=48, filesize=11.7 K 2024-12-04T19:15:55,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/B/883ae9d5e2a84ce1a45c75da78312688 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/883ae9d5e2a84ce1a45c75da78312688 2024-12-04T19:15:55,680 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/883ae9d5e2a84ce1a45c75da78312688, entries=150, sequenceid=48, filesize=11.7 K 2024-12-04T19:15:55,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/C/069acd4b31004462a3bf5807487fcbce as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/069acd4b31004462a3bf5807487fcbce 2024-12-04T19:15:55,684 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/069acd4b31004462a3bf5807487fcbce, entries=150, sequenceid=48, filesize=11.7 K 2024-12-04T19:15:55,685 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~127.08 KB/130125, heapSize ~364.10 KB/372840, currentSize=53.67 KB/54960 for 48c104a4c7a51d8b62d084d2a093043a in 1244ms, sequenceid=48, compaction requested=false 2024-12-04T19:15:55,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for 48c104a4c7a51d8b62d084d2a093043a: 2024-12-04T19:15:55,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:15:55,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-12-04T19:15:55,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-12-04T19:15:55,687 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-12-04T19:15:55,687 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6220 sec 2024-12-04T19:15:55,689 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 2.6310 sec 2024-12-04T19:15:55,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(8855): Flush requested on 48c104a4c7a51d8b62d084d2a093043a 2024-12-04T19:15:55,926 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 48c104a4c7a51d8b62d084d2a093043a 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-04T19:15:55,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 48c104a4c7a51d8b62d084d2a093043a, store=A 2024-12-04T19:15:55,927 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:15:55,927 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 48c104a4c7a51d8b62d084d2a093043a, store=B 2024-12-04T19:15:55,927 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:15:55,927 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 48c104a4c7a51d8b62d084d2a093043a, store=C 2024-12-04T19:15:55,927 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:15:55,929 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/A/c43a493531cb4af5bb9867891dc13df2 is 50, key is test_row_0/A:col10/1733339755925/Put/seqid=0 2024-12-04T19:15:55,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742278_1454 (size=14341) 2024-12-04T19:15:55,934 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/A/c43a493531cb4af5bb9867891dc13df2 2024-12-04T19:15:55,939 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/B/c0c41ec9f65e44b280436206b7d9e2f2 is 50, key is test_row_0/B:col10/1733339755925/Put/seqid=0 2024-12-04T19:15:55,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742279_1455 (size=12001) 2024-12-04T19:15:55,987 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:55,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52588 deadline: 1733339815976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:55,987 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:55,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52546 deadline: 1733339815977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:55,987 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:55,987 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:55,987 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:55,987 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:55,987 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:55,987 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:56,009 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-04T19:15:56,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:56,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52588 deadline: 1733339816092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:56,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:56,102 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:56,102 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:56,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52546 deadline: 1733339816092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:56,102 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:56,103 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:56,103 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:56,103 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:56,321 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:56,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52588 deadline: 1733339816311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:56,321 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:56,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:56,321 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:56,321 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:56,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52546 deadline: 1733339816312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:56,323 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:56,323 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:56,323 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:56,342 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/B/c0c41ec9f65e44b280436206b7d9e2f2 2024-12-04T19:15:56,351 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/C/c3b895109f6d43d58f4530f89d377e12 is 50, key is test_row_0/C:col10/1733339755925/Put/seqid=0 2024-12-04T19:15:56,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742280_1456 (size=12001) 2024-12-04T19:15:56,363 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/C/c3b895109f6d43d58f4530f89d377e12 2024-12-04T19:15:56,368 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/A/c43a493531cb4af5bb9867891dc13df2 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/c43a493531cb4af5bb9867891dc13df2 2024-12-04T19:15:56,374 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/c43a493531cb4af5bb9867891dc13df2, entries=200, sequenceid=61, filesize=14.0 K 2024-12-04T19:15:56,375 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/B/c0c41ec9f65e44b280436206b7d9e2f2 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/c0c41ec9f65e44b280436206b7d9e2f2 2024-12-04T19:15:56,383 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/c0c41ec9f65e44b280436206b7d9e2f2, entries=150, sequenceid=61, filesize=11.7 K 2024-12-04T19:15:56,384 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/C/c3b895109f6d43d58f4530f89d377e12 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/c3b895109f6d43d58f4530f89d377e12 2024-12-04T19:15:56,393 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/c3b895109f6d43d58f4530f89d377e12, entries=150, sequenceid=61, filesize=11.7 K 2024-12-04T19:15:56,394 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 48c104a4c7a51d8b62d084d2a093043a in 468ms, sequenceid=61, compaction requested=true 2024-12-04T19:15:56,394 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 48c104a4c7a51d8b62d084d2a093043a: 2024-12-04T19:15:56,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 48c104a4c7a51d8b62d084d2a093043a:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T19:15:56,394 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T19:15:56,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:15:56,395 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T19:15:56,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 48c104a4c7a51d8b62d084d2a093043a:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T19:15:56,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:15:56,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 48c104a4c7a51d8b62d084d2a093043a:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T19:15:56,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T19:15:56,396 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43023 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T19:15:56,396 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T19:15:56,396 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1541): 48c104a4c7a51d8b62d084d2a093043a/A is initiating minor compaction (all files) 2024-12-04T19:15:56,396 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1541): 48c104a4c7a51d8b62d084d2a093043a/B is initiating minor compaction (all files) 2024-12-04T19:15:56,396 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 48c104a4c7a51d8b62d084d2a093043a/B in TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:15:56,396 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 48c104a4c7a51d8b62d084d2a093043a/A in TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:15:56,396 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/9febb8cc21dd47f695ba56f16409c0aa, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/8909666709d74547b527b560d0e9903e, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/c43a493531cb4af5bb9867891dc13df2] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp, totalSize=42.0 K 2024-12-04T19:15:56,396 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/2e1082f0347e42afad92ca5855a1e4ed, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/883ae9d5e2a84ce1a45c75da78312688, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/c0c41ec9f65e44b280436206b7d9e2f2] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp, totalSize=35.2 K 2024-12-04T19:15:56,397 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9febb8cc21dd47f695ba56f16409c0aa, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733339753062 2024-12-04T19:15:56,397 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e1082f0347e42afad92ca5855a1e4ed, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733339753062 2024-12-04T19:15:56,397 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 883ae9d5e2a84ce1a45c75da78312688, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1733339753150 2024-12-04T19:15:56,397 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8909666709d74547b527b560d0e9903e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1733339753150 2024-12-04T19:15:56,397 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting c0c41ec9f65e44b280436206b7d9e2f2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1733339755277 2024-12-04T19:15:56,397 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting c43a493531cb4af5bb9867891dc13df2, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1733339755277 2024-12-04T19:15:56,405 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 48c104a4c7a51d8b62d084d2a093043a#A#compaction#380 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:15:56,405 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/A/cfc9b30527eb4443bee32764b34fe9b6 is 50, key is test_row_0/A:col10/1733339755925/Put/seqid=0 2024-12-04T19:15:56,406 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 48c104a4c7a51d8b62d084d2a093043a#B#compaction#381 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:15:56,406 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/B/8d7f81909fba4b99b9500dd98a2e81c7 is 50, key is test_row_0/B:col10/1733339755925/Put/seqid=0 2024-12-04T19:15:56,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742282_1458 (size=12104) 2024-12-04T19:15:56,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742281_1457 (size=12104) 2024-12-04T19:15:56,422 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/A/cfc9b30527eb4443bee32764b34fe9b6 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/cfc9b30527eb4443bee32764b34fe9b6 2024-12-04T19:15:56,430 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 48c104a4c7a51d8b62d084d2a093043a/A of 48c104a4c7a51d8b62d084d2a093043a into cfc9b30527eb4443bee32764b34fe9b6(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:15:56,430 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 48c104a4c7a51d8b62d084d2a093043a: 2024-12-04T19:15:56,430 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., storeName=48c104a4c7a51d8b62d084d2a093043a/A, priority=13, startTime=1733339756394; duration=0sec 2024-12-04T19:15:56,430 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T19:15:56,430 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 48c104a4c7a51d8b62d084d2a093043a:A 2024-12-04T19:15:56,430 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T19:15:56,431 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T19:15:56,431 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1541): 48c104a4c7a51d8b62d084d2a093043a/C is initiating minor compaction (all files) 2024-12-04T19:15:56,431 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 48c104a4c7a51d8b62d084d2a093043a/C in TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:15:56,432 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/6def83e5daaa470cad924c91e8427da0, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/069acd4b31004462a3bf5807487fcbce, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/c3b895109f6d43d58f4530f89d377e12] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp, totalSize=35.2 K 2024-12-04T19:15:56,432 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6def83e5daaa470cad924c91e8427da0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733339753062 2024-12-04T19:15:56,432 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 069acd4b31004462a3bf5807487fcbce, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1733339753150 2024-12-04T19:15:56,432 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting c3b895109f6d43d58f4530f89d377e12, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1733339755277 2024-12-04T19:15:56,445 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 48c104a4c7a51d8b62d084d2a093043a#C#compaction#382 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:15:56,447 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/C/53e7d43d46674abebc52d9962cc4c76c is 50, key is test_row_0/C:col10/1733339755925/Put/seqid=0 2024-12-04T19:15:56,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742283_1459 (size=12104) 2024-12-04T19:15:56,641 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 48c104a4c7a51d8b62d084d2a093043a 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-04T19:15:56,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 48c104a4c7a51d8b62d084d2a093043a, store=A 2024-12-04T19:15:56,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T19:15:56,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 48c104a4c7a51d8b62d084d2a093043a, store=B 2024-12-04T19:15:56,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T19:15:56,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 48c104a4c7a51d8b62d084d2a093043a, store=C 2024-12-04T19:15:56,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T19:15:56,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(8855): Flush requested on 48c104a4c7a51d8b62d084d2a093043a 2024-12-04T19:15:56,662 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/A/eb7c0d547f724ed19113db60a6d47ee3 is 50, key is test_row_0/A:col10/1733339756636/Put/seqid=0 2024-12-04T19:15:56,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742284_1460 (size=16681) 2024-12-04T19:15:56,685 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/A/eb7c0d547f724ed19113db60a6d47ee3 2024-12-04T19:15:56,700 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:56,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52546 deadline: 1733339816687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:56,701 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:56,701 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:56,701 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:56,707 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/B/8794873c6c724f65b5f7ade5865081bb is 50, key is test_row_0/B:col10/1733339756636/Put/seqid=0 2024-12-04T19:15:56,715 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:56,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52588 deadline: 1733339816699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:56,715 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:56,715 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:56,715 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:56,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742285_1461 (size=12001) 2024-12-04T19:15:56,819 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:56,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52546 deadline: 1733339816812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:56,820 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:56,821 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:56,821 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:56,821 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/B/8d7f81909fba4b99b9500dd98a2e81c7 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/8d7f81909fba4b99b9500dd98a2e81c7 2024-12-04T19:15:56,825 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 48c104a4c7a51d8b62d084d2a093043a/B of 48c104a4c7a51d8b62d084d2a093043a into 8d7f81909fba4b99b9500dd98a2e81c7(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:15:56,825 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 48c104a4c7a51d8b62d084d2a093043a: 2024-12-04T19:15:56,825 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., storeName=48c104a4c7a51d8b62d084d2a093043a/B, priority=13, startTime=1733339756395; duration=0sec 2024-12-04T19:15:56,825 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:15:56,825 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 48c104a4c7a51d8b62d084d2a093043a:B 2024-12-04T19:15:56,839 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:56,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52588 deadline: 1733339816822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:56,840 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:56,840 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:56,840 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:56,868 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/C/53e7d43d46674abebc52d9962cc4c76c as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/53e7d43d46674abebc52d9962cc4c76c 2024-12-04T19:15:56,876 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 48c104a4c7a51d8b62d084d2a093043a/C of 48c104a4c7a51d8b62d084d2a093043a into 53e7d43d46674abebc52d9962cc4c76c(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:15:56,876 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 48c104a4c7a51d8b62d084d2a093043a: 2024-12-04T19:15:56,876 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., storeName=48c104a4c7a51d8b62d084d2a093043a/C, priority=13, startTime=1733339756395; duration=0sec 2024-12-04T19:15:56,876 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:15:56,876 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 48c104a4c7a51d8b62d084d2a093043a:C 2024-12-04T19:15:57,041 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:57,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52546 deadline: 1733339817033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:57,041 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:57,041 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:57,041 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:57,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:57,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52588 deadline: 1733339817043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:57,054 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:57,054 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:57,054 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:57,124 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/B/8794873c6c724f65b5f7ade5865081bb 2024-12-04T19:15:57,146 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/C/ab4db85ae34a455591fd269cb6623d08 is 50, key is test_row_0/C:col10/1733339756636/Put/seqid=0 2024-12-04T19:15:57,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742286_1462 (size=12001) 2024-12-04T19:15:57,191 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/C/ab4db85ae34a455591fd269cb6623d08 2024-12-04T19:15:57,198 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/A/eb7c0d547f724ed19113db60a6d47ee3 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/eb7c0d547f724ed19113db60a6d47ee3 2024-12-04T19:15:57,203 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/eb7c0d547f724ed19113db60a6d47ee3, entries=250, sequenceid=83, filesize=16.3 K 2024-12-04T19:15:57,204 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/B/8794873c6c724f65b5f7ade5865081bb as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/8794873c6c724f65b5f7ade5865081bb 2024-12-04T19:15:57,215 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/8794873c6c724f65b5f7ade5865081bb, entries=150, sequenceid=83, filesize=11.7 K 2024-12-04T19:15:57,216 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/C/ab4db85ae34a455591fd269cb6623d08 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/ab4db85ae34a455591fd269cb6623d08 2024-12-04T19:15:57,229 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/ab4db85ae34a455591fd269cb6623d08, entries=150, sequenceid=83, filesize=11.7 K 2024-12-04T19:15:57,230 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 48c104a4c7a51d8b62d084d2a093043a in 589ms, sequenceid=83, compaction requested=false 2024-12-04T19:15:57,230 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 48c104a4c7a51d8b62d084d2a093043a: 2024-12-04T19:15:57,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-04T19:15:57,232 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T19:15:57,233 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T19:15:57,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-12-04T19:15:57,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-04T19:15:57,235 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T19:15:57,236 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T19:15:57,236 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T19:15:57,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-04T19:15:57,360 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 48c104a4c7a51d8b62d084d2a093043a 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-04T19:15:57,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(8855): Flush requested on 48c104a4c7a51d8b62d084d2a093043a 2024-12-04T19:15:57,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 48c104a4c7a51d8b62d084d2a093043a, store=A 2024-12-04T19:15:57,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:15:57,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 48c104a4c7a51d8b62d084d2a093043a, store=B 2024-12-04T19:15:57,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:15:57,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 48c104a4c7a51d8b62d084d2a093043a, store=C 2024-12-04T19:15:57,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:15:57,366 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/A/57da3c4c57bf4373826531b6d7f7b684 is 50, key is test_row_0/A:col10/1733339756666/Put/seqid=0 2024-12-04T19:15:57,389 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-04T19:15:57,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:15:57,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. as already flushing 2024-12-04T19:15:57,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:15:57,390 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:57,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:57,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742287_1463 (size=16681) 2024-12-04T19:15:57,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:57,409 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:57,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52494 deadline: 1733339817391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:57,409 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:57,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52544 deadline: 1733339817392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:57,410 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:57,410 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:57,410 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:57,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52588 deadline: 1733339817394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:57,410 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:57,410 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:57,410 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:57,410 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:57,410 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:57,410 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:57,410 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:57,411 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:57,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52546 deadline: 1733339817400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:57,411 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:57,411 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:57,411 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:57,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:57,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52496 deadline: 1733339817409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:57,424 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:57,424 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:57,424 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:57,517 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:57,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52544 deadline: 1733339817512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:57,518 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:57,518 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:57,518 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:57,530 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:57,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52494 deadline: 1733339817523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:57,531 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:57,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52588 deadline: 1733339817523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:57,531 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:57,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52546 deadline: 1733339817523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:57,532 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:57,532 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:57,532 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:57,532 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:57,532 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:57,532 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:57,533 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:57,533 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:57,533 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:57,537 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:57,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52496 deadline: 1733339817533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:57,539 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:57,539 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:57,539 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:57,543 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-04T19:15:57,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:15:57,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. as already flushing 2024-12-04T19:15:57,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:15:57,544 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:57,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:57,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:57,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-04T19:15:57,698 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-04T19:15:57,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:15:57,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. as already flushing 2024-12-04T19:15:57,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:15:57,698 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:57,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:57,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:57,729 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:57,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52544 deadline: 1733339817722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:57,729 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:57,729 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:57,729 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:57,745 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:57,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52494 deadline: 1733339817742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:57,745 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:57,745 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:57,745 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:57,745 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:57,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52546 deadline: 1733339817742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:57,746 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:57,746 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:57,745 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:57,746 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:57,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52588 deadline: 1733339817742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:57,746 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:57,746 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:57,746 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:57,746 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:57,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52496 deadline: 1733339817742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:57,746 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:57,746 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:57,746 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:57,790 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/A/57da3c4c57bf4373826531b6d7f7b684 2024-12-04T19:15:57,796 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/B/8d6d4277649c48aca8e1bc5b7b3ecc0c is 50, key is test_row_0/B:col10/1733339756666/Put/seqid=0 2024-12-04T19:15:57,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742288_1464 (size=12001) 2024-12-04T19:15:57,852 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-04T19:15:57,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:15:57,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. as already flushing 2024-12-04T19:15:57,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:15:57,852 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:57,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:57,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:57,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-04T19:15:58,004 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-04T19:15:58,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:15:58,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. as already flushing 2024-12-04T19:15:58,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:15:58,005 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:58,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:58,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:58,043 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:58,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52544 deadline: 1733339818042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:58,043 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:58,043 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:58,043 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:58,057 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:58,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52494 deadline: 1733339818051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:58,057 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:58,057 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:58,057 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:58,057 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:58,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52546 deadline: 1733339818051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:58,057 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:58,057 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:58,057 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:58,057 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:58,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52588 deadline: 1733339818051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:58,058 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:58,058 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:58,058 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:58,058 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:58,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52496 deadline: 1733339818051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:58,058 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:58,058 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:58,058 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:58,157 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-04T19:15:58,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:15:58,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. as already flushing 2024-12-04T19:15:58,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:15:58,157 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:58,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:58,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:58,199 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/B/8d6d4277649c48aca8e1bc5b7b3ecc0c 2024-12-04T19:15:58,205 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/C/930375700cc24b8799a269f65c50da26 is 50, key is test_row_0/C:col10/1733339756666/Put/seqid=0 2024-12-04T19:15:58,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742289_1465 (size=12001) 2024-12-04T19:15:58,309 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-04T19:15:58,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:15:58,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. as already flushing 2024-12-04T19:15:58,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:15:58,310 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:58,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:58,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:58,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-04T19:15:58,462 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-04T19:15:58,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:15:58,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. as already flushing 2024-12-04T19:15:58,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:15:58,462 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:58,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:58,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:58,554 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:58,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52544 deadline: 1733339818552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:58,555 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:58,555 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:58,555 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:58,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:58,567 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:58,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52494 deadline: 1733339818561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:58,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52546 deadline: 1733339818561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:58,567 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:58,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:58,567 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:58,567 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:58,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52588 deadline: 1733339818561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:58,568 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:58,568 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:58,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52496 deadline: 1733339818562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:58,568 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:58,568 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:58,568 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:58,568 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:58,568 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:58,568 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:58,568 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:58,568 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:58,615 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-04T19:15:58,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:15:58,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. as already flushing 2024-12-04T19:15:58,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:15:58,615 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:58,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:58,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:15:58,618 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/C/930375700cc24b8799a269f65c50da26 2024-12-04T19:15:58,621 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/A/57da3c4c57bf4373826531b6d7f7b684 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/57da3c4c57bf4373826531b6d7f7b684 2024-12-04T19:15:58,624 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/57da3c4c57bf4373826531b6d7f7b684, entries=250, sequenceid=98, filesize=16.3 K 2024-12-04T19:15:58,624 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/B/8d6d4277649c48aca8e1bc5b7b3ecc0c as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/8d6d4277649c48aca8e1bc5b7b3ecc0c 2024-12-04T19:15:58,627 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/8d6d4277649c48aca8e1bc5b7b3ecc0c, entries=150, sequenceid=98, filesize=11.7 K 2024-12-04T19:15:58,627 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/C/930375700cc24b8799a269f65c50da26 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/930375700cc24b8799a269f65c50da26 2024-12-04T19:15:58,630 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/930375700cc24b8799a269f65c50da26, entries=150, sequenceid=98, filesize=11.7 K 2024-12-04T19:15:58,630 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 48c104a4c7a51d8b62d084d2a093043a in 1270ms, sequenceid=98, compaction requested=true 2024-12-04T19:15:58,630 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 48c104a4c7a51d8b62d084d2a093043a: 2024-12-04T19:15:58,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 48c104a4c7a51d8b62d084d2a093043a:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T19:15:58,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:15:58,631 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T19:15:58,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 48c104a4c7a51d8b62d084d2a093043a:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T19:15:58,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:15:58,631 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T19:15:58,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 48c104a4c7a51d8b62d084d2a093043a:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T19:15:58,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T19:15:58,631 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 45466 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T19:15:58,631 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T19:15:58,631 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1541): 48c104a4c7a51d8b62d084d2a093043a/A is initiating minor compaction (all files) 2024-12-04T19:15:58,631 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1541): 48c104a4c7a51d8b62d084d2a093043a/B is initiating minor compaction (all files) 2024-12-04T19:15:58,631 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 48c104a4c7a51d8b62d084d2a093043a/B in TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:15:58,631 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 48c104a4c7a51d8b62d084d2a093043a/A in TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:15:58,631 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/cfc9b30527eb4443bee32764b34fe9b6, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/eb7c0d547f724ed19113db60a6d47ee3, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/57da3c4c57bf4373826531b6d7f7b684] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp, totalSize=44.4 K 2024-12-04T19:15:58,631 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/8d7f81909fba4b99b9500dd98a2e81c7, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/8794873c6c724f65b5f7ade5865081bb, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/8d6d4277649c48aca8e1bc5b7b3ecc0c] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp, totalSize=35.3 K 2024-12-04T19:15:58,632 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d7f81909fba4b99b9500dd98a2e81c7, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1733339755277 2024-12-04T19:15:58,632 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting cfc9b30527eb4443bee32764b34fe9b6, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1733339755277 2024-12-04T19:15:58,632 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 8794873c6c724f65b5f7ade5865081bb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1733339755971 2024-12-04T19:15:58,632 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting eb7c0d547f724ed19113db60a6d47ee3, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1733339755971 2024-12-04T19:15:58,632 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d6d4277649c48aca8e1bc5b7b3ecc0c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1733339756666 2024-12-04T19:15:58,632 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 57da3c4c57bf4373826531b6d7f7b684, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1733339756666 2024-12-04T19:15:58,637 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 48c104a4c7a51d8b62d084d2a093043a#B#compaction#389 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:15:58,637 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/B/f2b361ac91034406943633ea9ba76810 is 50, key is test_row_0/B:col10/1733339756666/Put/seqid=0 2024-12-04T19:15:58,637 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 48c104a4c7a51d8b62d084d2a093043a#A#compaction#390 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:15:58,638 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/A/33dbdd994f0640e08417ec2494dcfc82 is 50, key is test_row_0/A:col10/1733339756666/Put/seqid=0 2024-12-04T19:15:58,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742290_1466 (size=12207) 2024-12-04T19:15:58,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742291_1467 (size=12207) 2024-12-04T19:15:58,767 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-04T19:15:58,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:15:58,767 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing 48c104a4c7a51d8b62d084d2a093043a 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-04T19:15:58,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 48c104a4c7a51d8b62d084d2a093043a, store=A 2024-12-04T19:15:58,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:15:58,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 48c104a4c7a51d8b62d084d2a093043a, store=B 2024-12-04T19:15:58,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:15:58,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 48c104a4c7a51d8b62d084d2a093043a, store=C 2024-12-04T19:15:58,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:15:58,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/A/bd9707808aaf42808faed48fd509f8d1 is 50, key is test_row_0/A:col10/1733339757403/Put/seqid=0 2024-12-04T19:15:58,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742292_1468 (size=12001) 2024-12-04T19:15:59,048 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/B/f2b361ac91034406943633ea9ba76810 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/f2b361ac91034406943633ea9ba76810 2024-12-04T19:15:59,052 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 48c104a4c7a51d8b62d084d2a093043a/B of 48c104a4c7a51d8b62d084d2a093043a into f2b361ac91034406943633ea9ba76810(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:15:59,052 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 48c104a4c7a51d8b62d084d2a093043a: 2024-12-04T19:15:59,052 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., storeName=48c104a4c7a51d8b62d084d2a093043a/B, priority=13, startTime=1733339758631; duration=0sec 2024-12-04T19:15:59,052 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T19:15:59,052 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 48c104a4c7a51d8b62d084d2a093043a:B 2024-12-04T19:15:59,052 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T19:15:59,054 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/A/33dbdd994f0640e08417ec2494dcfc82 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/33dbdd994f0640e08417ec2494dcfc82 2024-12-04T19:15:59,055 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T19:15:59,055 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1541): 48c104a4c7a51d8b62d084d2a093043a/C is initiating minor compaction (all files) 2024-12-04T19:15:59,055 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 48c104a4c7a51d8b62d084d2a093043a/C in TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:15:59,055 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/53e7d43d46674abebc52d9962cc4c76c, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/ab4db85ae34a455591fd269cb6623d08, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/930375700cc24b8799a269f65c50da26] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp, totalSize=35.3 K 2024-12-04T19:15:59,055 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 53e7d43d46674abebc52d9962cc4c76c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1733339755277 2024-12-04T19:15:59,056 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting ab4db85ae34a455591fd269cb6623d08, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1733339755971 2024-12-04T19:15:59,060 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 930375700cc24b8799a269f65c50da26, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1733339756666 2024-12-04T19:15:59,063 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 48c104a4c7a51d8b62d084d2a093043a/A of 48c104a4c7a51d8b62d084d2a093043a into 33dbdd994f0640e08417ec2494dcfc82(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:15:59,063 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 48c104a4c7a51d8b62d084d2a093043a: 2024-12-04T19:15:59,063 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., storeName=48c104a4c7a51d8b62d084d2a093043a/A, priority=13, startTime=1733339758630; duration=0sec 2024-12-04T19:15:59,063 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:15:59,063 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 48c104a4c7a51d8b62d084d2a093043a:A 2024-12-04T19:15:59,066 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 48c104a4c7a51d8b62d084d2a093043a#C#compaction#392 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:15:59,066 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/C/836a3d5aacae47de830f48f1faa051b5 is 50, key is test_row_0/C:col10/1733339756666/Put/seqid=0 2024-12-04T19:15:59,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742293_1469 (size=12207) 2024-12-04T19:15:59,176 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/A/bd9707808aaf42808faed48fd509f8d1 2024-12-04T19:15:59,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/B/bc0512c2a1bd4cb787b397d9f2d1002f is 50, key is test_row_0/B:col10/1733339757403/Put/seqid=0 2024-12-04T19:15:59,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742294_1470 (size=12001) 2024-12-04T19:15:59,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-04T19:15:59,477 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/C/836a3d5aacae47de830f48f1faa051b5 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/836a3d5aacae47de830f48f1faa051b5 2024-12-04T19:15:59,480 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 48c104a4c7a51d8b62d084d2a093043a/C of 48c104a4c7a51d8b62d084d2a093043a into 836a3d5aacae47de830f48f1faa051b5(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:15:59,480 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 48c104a4c7a51d8b62d084d2a093043a: 2024-12-04T19:15:59,480 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., storeName=48c104a4c7a51d8b62d084d2a093043a/C, priority=13, startTime=1733339758631; duration=0sec 2024-12-04T19:15:59,480 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:15:59,480 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 48c104a4c7a51d8b62d084d2a093043a:C 2024-12-04T19:15:59,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(8855): Flush requested on 48c104a4c7a51d8b62d084d2a093043a 2024-12-04T19:15:59,575 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. as already flushing 2024-12-04T19:15:59,586 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/B/bc0512c2a1bd4cb787b397d9f2d1002f 2024-12-04T19:15:59,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/C/ca43ec72a6c842ce911de40021f9b054 is 50, key is test_row_0/C:col10/1733339757403/Put/seqid=0 2024-12-04T19:15:59,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742295_1471 (size=12001) 2024-12-04T19:15:59,594 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:59,595 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:59,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52588 deadline: 1733339819587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:59,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52496 deadline: 1733339819587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:59,595 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:59,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52544 deadline: 1733339819588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:59,595 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:59,595 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:59,595 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:59,595 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:59,595 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:59,595 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:59,595 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:59,595 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:59,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52494 deadline: 1733339819591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:59,595 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:59,595 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:59,595 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:59,595 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:59,596 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:59,597 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:59,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52546 deadline: 1733339819595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:59,597 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:59,597 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:59,597 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:59,706 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:59,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52588 deadline: 1733339819702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:59,706 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:59,706 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:59,706 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:59,706 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:59,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52496 deadline: 1733339819702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:59,706 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:59,706 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:59,706 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:59,706 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:59,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52544 deadline: 1733339819702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:59,707 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:59,707 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:59,707 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:59,707 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:59,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52494 deadline: 1733339819702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:59,707 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:59,707 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:59,707 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:59,707 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:59,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52546 deadline: 1733339819702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:59,707 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:59,707 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:59,707 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:59,916 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:59,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52588 deadline: 1733339819912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:59,916 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:59,916 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:59,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52496 deadline: 1733339819912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:59,916 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:59,916 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:59,916 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:59,916 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:59,917 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:59,917 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:59,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52544 deadline: 1733339819912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:59,917 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:59,917 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:59,917 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:59,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52494 deadline: 1733339819912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:59,917 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:59,917 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:59,917 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:15:59,917 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:59,917 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:59,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52546 deadline: 1733339819912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:15:59,917 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:59,917 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:15:59,917 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:15:59,995 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/C/ca43ec72a6c842ce911de40021f9b054 2024-12-04T19:15:59,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/A/bd9707808aaf42808faed48fd509f8d1 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/bd9707808aaf42808faed48fd509f8d1 2024-12-04T19:16:00,000 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/bd9707808aaf42808faed48fd509f8d1, entries=150, sequenceid=117, filesize=11.7 K 2024-12-04T19:16:00,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/B/bc0512c2a1bd4cb787b397d9f2d1002f as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/bc0512c2a1bd4cb787b397d9f2d1002f 2024-12-04T19:16:00,005 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/bc0512c2a1bd4cb787b397d9f2d1002f, entries=150, sequenceid=117, filesize=11.7 K 2024-12-04T19:16:00,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/C/ca43ec72a6c842ce911de40021f9b054 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/ca43ec72a6c842ce911de40021f9b054 2024-12-04T19:16:00,008 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/ca43ec72a6c842ce911de40021f9b054, entries=150, sequenceid=117, filesize=11.7 K 2024-12-04T19:16:00,009 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 48c104a4c7a51d8b62d084d2a093043a in 1242ms, sequenceid=117, compaction requested=false 2024-12-04T19:16:00,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for 48c104a4c7a51d8b62d084d2a093043a: 2024-12-04T19:16:00,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:16:00,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-12-04T19:16:00,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-12-04T19:16:00,011 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-12-04T19:16:00,011 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7740 sec 2024-12-04T19:16:00,012 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 2.7790 sec 2024-12-04T19:16:00,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(8855): Flush requested on 48c104a4c7a51d8b62d084d2a093043a 2024-12-04T19:16:00,227 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 48c104a4c7a51d8b62d084d2a093043a 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-04T19:16:00,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 48c104a4c7a51d8b62d084d2a093043a, store=A 2024-12-04T19:16:00,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:16:00,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 48c104a4c7a51d8b62d084d2a093043a, store=B 2024-12-04T19:16:00,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:16:00,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 48c104a4c7a51d8b62d084d2a093043a, store=C 2024-12-04T19:16:00,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:16:00,232 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/A/9b435a62e9b749f88182df453e4c895e is 50, key is test_row_0/A:col10/1733339760225/Put/seqid=0 2024-12-04T19:16:00,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742296_1472 (size=16831) 2024-12-04T19:16:00,259 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:00,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52496 deadline: 1733339820249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:00,259 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:00,259 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:00,260 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:00,260 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:00,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52494 deadline: 1733339820250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:00,260 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:00,260 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:00,260 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:00,263 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:00,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52544 deadline: 1733339820253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:00,263 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:00,263 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:00,263 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:00,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52546 deadline: 1733339820253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:00,263 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:00,263 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:00,263 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:00,263 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:00,263 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:00,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52588 deadline: 1733339820259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:00,264 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:00,264 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:00,264 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:00,366 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:00,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52496 deadline: 1733339820361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:00,366 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:00,366 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:00,366 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:00,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52494 deadline: 1733339820361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:00,366 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:00,366 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:00,367 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:00,367 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:00,375 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:00,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52544 deadline: 1733339820372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:00,375 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:00,375 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:00,375 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:00,375 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:00,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52546 deadline: 1733339820372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:00,375 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:00,375 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:00,375 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:00,375 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:00,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52588 deadline: 1733339820372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:00,376 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:00,376 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:00,376 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:00,577 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:00,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52496 deadline: 1733339820571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:00,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:00,578 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:00,578 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:00,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52494 deadline: 1733339820572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:00,578 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:00,578 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:00,578 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:00,578 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:00,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:00,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52544 deadline: 1733339820582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:00,587 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:00,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:00,587 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:00,587 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:00,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52546 deadline: 1733339820582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:00,588 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:00,588 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:00,588 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:00,588 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:00,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52588 deadline: 1733339820582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:00,588 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:00,588 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:00,588 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:00,644 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/A/9b435a62e9b749f88182df453e4c895e 2024-12-04T19:16:00,648 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/B/dc4a5b7de29446ac9b97c5e7fe135d67 is 50, key is test_row_0/B:col10/1733339760225/Put/seqid=0 2024-12-04T19:16:00,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742297_1473 (size=12151) 2024-12-04T19:16:00,654 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/B/dc4a5b7de29446ac9b97c5e7fe135d67 2024-12-04T19:16:00,659 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/C/6fea85c990d34e8ba39061661e359986 is 50, key is test_row_0/C:col10/1733339760225/Put/seqid=0 2024-12-04T19:16:00,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742298_1474 (size=12151) 2024-12-04T19:16:00,888 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:00,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52496 deadline: 1733339820881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:00,888 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:00,888 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:00,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52494 deadline: 1733339820881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:00,888 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:00,888 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:00,888 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:00,888 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:00,888 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:00,896 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:00,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52544 deadline: 1733339820891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:00,896 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:00,896 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:00,896 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:00,896 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:00,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52546 deadline: 1733339820892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:00,896 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:00,896 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:00,896 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:00,906 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:00,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52588 deadline: 1733339820902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:00,906 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:00,906 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:00,906 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:01,062 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/C/6fea85c990d34e8ba39061661e359986 2024-12-04T19:16:01,065 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/A/9b435a62e9b749f88182df453e4c895e as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/9b435a62e9b749f88182df453e4c895e 2024-12-04T19:16:01,068 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/9b435a62e9b749f88182df453e4c895e, entries=250, sequenceid=135, filesize=16.4 K 2024-12-04T19:16:01,069 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/B/dc4a5b7de29446ac9b97c5e7fe135d67 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/dc4a5b7de29446ac9b97c5e7fe135d67 2024-12-04T19:16:01,072 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/dc4a5b7de29446ac9b97c5e7fe135d67, entries=150, sequenceid=135, filesize=11.9 K 2024-12-04T19:16:01,072 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/C/6fea85c990d34e8ba39061661e359986 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/6fea85c990d34e8ba39061661e359986 2024-12-04T19:16:01,075 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/6fea85c990d34e8ba39061661e359986, entries=150, sequenceid=135, filesize=11.9 K 2024-12-04T19:16:01,076 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 48c104a4c7a51d8b62d084d2a093043a in 850ms, sequenceid=135, compaction requested=true 2024-12-04T19:16:01,076 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 48c104a4c7a51d8b62d084d2a093043a: 2024-12-04T19:16:01,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 48c104a4c7a51d8b62d084d2a093043a:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T19:16:01,076 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T19:16:01,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:16:01,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 48c104a4c7a51d8b62d084d2a093043a:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T19:16:01,076 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T19:16:01,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:16:01,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 48c104a4c7a51d8b62d084d2a093043a:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T19:16:01,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T19:16:01,077 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41039 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T19:16:01,077 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T19:16:01,077 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1541): 48c104a4c7a51d8b62d084d2a093043a/A is initiating minor compaction (all files) 2024-12-04T19:16:01,077 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1541): 48c104a4c7a51d8b62d084d2a093043a/B is initiating minor compaction (all files) 2024-12-04T19:16:01,077 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 48c104a4c7a51d8b62d084d2a093043a/B in TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:16:01,077 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 48c104a4c7a51d8b62d084d2a093043a/A in TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:16:01,077 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/f2b361ac91034406943633ea9ba76810, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/bc0512c2a1bd4cb787b397d9f2d1002f, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/dc4a5b7de29446ac9b97c5e7fe135d67] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp, totalSize=35.5 K 2024-12-04T19:16:01,077 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/33dbdd994f0640e08417ec2494dcfc82, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/bd9707808aaf42808faed48fd509f8d1, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/9b435a62e9b749f88182df453e4c895e] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp, totalSize=40.1 K 2024-12-04T19:16:01,077 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 33dbdd994f0640e08417ec2494dcfc82, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1733339756666 2024-12-04T19:16:01,077 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting f2b361ac91034406943633ea9ba76810, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1733339756666 2024-12-04T19:16:01,078 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting bc0512c2a1bd4cb787b397d9f2d1002f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1733339757389 2024-12-04T19:16:01,078 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting bd9707808aaf42808faed48fd509f8d1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1733339757389 2024-12-04T19:16:01,078 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b435a62e9b749f88182df453e4c895e, keycount=250, bloomtype=ROW, size=16.4 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1733339759584 2024-12-04T19:16:01,078 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting dc4a5b7de29446ac9b97c5e7fe135d67, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1733339759593 2024-12-04T19:16:01,084 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 48c104a4c7a51d8b62d084d2a093043a#B#compaction#398 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:16:01,084 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/B/11a6907ef0c64f44b73d9a4f5f8b9b65 is 50, key is test_row_0/B:col10/1733339760225/Put/seqid=0 2024-12-04T19:16:01,095 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 48c104a4c7a51d8b62d084d2a093043a#A#compaction#399 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:16:01,096 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/A/df90f1337ba54599bd1c10c45803e2f7 is 50, key is test_row_0/A:col10/1733339760225/Put/seqid=0 2024-12-04T19:16:01,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742299_1475 (size=12459) 2024-12-04T19:16:01,101 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/B/11a6907ef0c64f44b73d9a4f5f8b9b65 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/11a6907ef0c64f44b73d9a4f5f8b9b65 2024-12-04T19:16:01,105 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 48c104a4c7a51d8b62d084d2a093043a/B of 48c104a4c7a51d8b62d084d2a093043a into 11a6907ef0c64f44b73d9a4f5f8b9b65(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:16:01,105 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 48c104a4c7a51d8b62d084d2a093043a: 2024-12-04T19:16:01,105 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., storeName=48c104a4c7a51d8b62d084d2a093043a/B, priority=13, startTime=1733339761076; duration=0sec 2024-12-04T19:16:01,105 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T19:16:01,105 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 48c104a4c7a51d8b62d084d2a093043a:B 2024-12-04T19:16:01,105 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T19:16:01,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742300_1476 (size=12459) 2024-12-04T19:16:01,106 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T19:16:01,107 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1541): 48c104a4c7a51d8b62d084d2a093043a/C is initiating minor compaction (all files) 2024-12-04T19:16:01,107 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 48c104a4c7a51d8b62d084d2a093043a/C in TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:16:01,107 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/836a3d5aacae47de830f48f1faa051b5, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/ca43ec72a6c842ce911de40021f9b054, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/6fea85c990d34e8ba39061661e359986] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp, totalSize=35.5 K 2024-12-04T19:16:01,107 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 836a3d5aacae47de830f48f1faa051b5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1733339756666 2024-12-04T19:16:01,108 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting ca43ec72a6c842ce911de40021f9b054, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1733339757389 2024-12-04T19:16:01,108 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 6fea85c990d34e8ba39061661e359986, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1733339759593 2024-12-04T19:16:01,113 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 48c104a4c7a51d8b62d084d2a093043a#C#compaction#400 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:16:01,114 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/C/cda0c3296f8a4bf18b36f030e8a76ec1 is 50, key is test_row_0/C:col10/1733339760225/Put/seqid=0 2024-12-04T19:16:01,115 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/A/df90f1337ba54599bd1c10c45803e2f7 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/df90f1337ba54599bd1c10c45803e2f7 2024-12-04T19:16:01,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742301_1477 (size=12459) 2024-12-04T19:16:01,119 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 48c104a4c7a51d8b62d084d2a093043a/A of 48c104a4c7a51d8b62d084d2a093043a into df90f1337ba54599bd1c10c45803e2f7(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:16:01,119 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 48c104a4c7a51d8b62d084d2a093043a: 2024-12-04T19:16:01,119 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., storeName=48c104a4c7a51d8b62d084d2a093043a/A, priority=13, startTime=1733339761076; duration=0sec 2024-12-04T19:16:01,119 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:16:01,119 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 48c104a4c7a51d8b62d084d2a093043a:A 2024-12-04T19:16:01,122 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/C/cda0c3296f8a4bf18b36f030e8a76ec1 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/cda0c3296f8a4bf18b36f030e8a76ec1 2024-12-04T19:16:01,126 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 48c104a4c7a51d8b62d084d2a093043a/C of 48c104a4c7a51d8b62d084d2a093043a into cda0c3296f8a4bf18b36f030e8a76ec1(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:16:01,126 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 48c104a4c7a51d8b62d084d2a093043a: 2024-12-04T19:16:01,126 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., storeName=48c104a4c7a51d8b62d084d2a093043a/C, priority=13, startTime=1733339761076; duration=0sec 2024-12-04T19:16:01,126 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:16:01,126 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 48c104a4c7a51d8b62d084d2a093043a:C 2024-12-04T19:16:01,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-04T19:16:01,392 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T19:16:01,392 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T19:16:01,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-12-04T19:16:01,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-04T19:16:01,394 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T19:16:01,394 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T19:16:01,395 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T19:16:01,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(8855): Flush requested on 48c104a4c7a51d8b62d084d2a093043a 2024-12-04T19:16:01,407 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 48c104a4c7a51d8b62d084d2a093043a 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-04T19:16:01,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 48c104a4c7a51d8b62d084d2a093043a, store=A 2024-12-04T19:16:01,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:16:01,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 48c104a4c7a51d8b62d084d2a093043a, store=B 2024-12-04T19:16:01,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:16:01,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 48c104a4c7a51d8b62d084d2a093043a, store=C 2024-12-04T19:16:01,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:16:01,411 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/A/db79c289b4854195b97f58a565038847 is 50, key is test_row_0/A:col10/1733339760248/Put/seqid=0 2024-12-04T19:16:01,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742302_1478 (size=16931) 2024-12-04T19:16:01,429 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:01,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52544 deadline: 1733339821420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:01,430 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:01,430 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:01,430 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:01,430 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:01,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52496 deadline: 1733339821421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:01,430 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:01,430 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:01,430 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:01,430 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:01,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52546 deadline: 1733339821425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:01,431 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:01,431 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:01,431 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:01,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:01,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52588 deadline: 1733339821426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:01,434 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:01,434 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:01,434 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:01,434 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:01,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52494 deadline: 1733339821427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:01,434 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:01,434 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:01,434 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:01,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-04T19:16:01,539 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:01,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52496 deadline: 1733339821531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:01,539 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:01,539 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:01,540 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:01,547 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-04T19:16:01,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:16:01,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. as already flushing 2024-12-04T19:16:01,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:16:01,547 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:01,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:01,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:01,548 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:01,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52544 deadline: 1733339821541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:01,549 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:01,549 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:01,549 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:01,549 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:01,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52546 deadline: 1733339821542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:01,549 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:01,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52588 deadline: 1733339821542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:01,549 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:01,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52494 deadline: 1733339821542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:01,549 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:01,549 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:01,549 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:01,550 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:01,550 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:01,551 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:01,551 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:01,551 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:01,551 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:01,700 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-04T19:16:01,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:16:01,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. as already flushing 2024-12-04T19:16:01,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:16:01,700 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:01,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:01,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:01,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-04T19:16:01,757 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:01,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52496 deadline: 1733339821752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:01,757 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:01,757 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:01,757 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:01,757 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:01,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52544 deadline: 1733339821752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:01,758 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:01,758 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:01,758 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:01,758 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:01,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52546 deadline: 1733339821752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:01,758 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:01,758 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:01,758 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:01,766 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:01,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52588 deadline: 1733339821761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:01,767 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:01,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52494 deadline: 1733339821761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:01,767 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:01,767 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:01,767 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:01,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:01,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:01,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:01,814 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/A/db79c289b4854195b97f58a565038847 2024-12-04T19:16:01,819 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/B/80252d0e2ca6455ba0189160b4ed543c is 50, key is test_row_0/B:col10/1733339760248/Put/seqid=0 2024-12-04T19:16:01,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742303_1479 (size=12151) 2024-12-04T19:16:01,853 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-04T19:16:01,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:16:01,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. as already flushing 2024-12-04T19:16:01,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:16:01,853 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:01,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:01,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:02,005 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-04T19:16:02,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:16:02,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. as already flushing 2024-12-04T19:16:02,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:16:02,006 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:02,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:02,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:02,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-04T19:16:02,065 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:02,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52496 deadline: 1733339822062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:02,066 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:02,066 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:02,066 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:02,066 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:02,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52544 deadline: 1733339822062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:02,066 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:02,066 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:02,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52546 deadline: 1733339822062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:02,066 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:02,066 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:02,067 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:02,067 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:02,067 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:02,073 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:02,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52588 deadline: 1733339822072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:02,073 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:02,073 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:02,073 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:02,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52494 deadline: 1733339822072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:02,073 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:02,073 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:02,073 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:02,073 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:02,158 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-04T19:16:02,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:16:02,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. as already flushing 2024-12-04T19:16:02,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:16:02,158 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:02,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:02,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:02,223 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/B/80252d0e2ca6455ba0189160b4ed543c 2024-12-04T19:16:02,229 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/C/baa05d747946429a8756ac1bb3126219 is 50, key is test_row_0/C:col10/1733339760248/Put/seqid=0 2024-12-04T19:16:02,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742304_1480 (size=12151) 2024-12-04T19:16:02,238 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/C/baa05d747946429a8756ac1bb3126219 2024-12-04T19:16:02,245 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/A/db79c289b4854195b97f58a565038847 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/db79c289b4854195b97f58a565038847 2024-12-04T19:16:02,248 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/db79c289b4854195b97f58a565038847, entries=250, sequenceid=157, filesize=16.5 K 2024-12-04T19:16:02,251 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/B/80252d0e2ca6455ba0189160b4ed543c as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/80252d0e2ca6455ba0189160b4ed543c 2024-12-04T19:16:02,256 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/80252d0e2ca6455ba0189160b4ed543c, entries=150, sequenceid=157, filesize=11.9 K 2024-12-04T19:16:02,257 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/C/baa05d747946429a8756ac1bb3126219 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/baa05d747946429a8756ac1bb3126219 2024-12-04T19:16:02,261 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/baa05d747946429a8756ac1bb3126219, entries=150, sequenceid=157, filesize=11.9 K 2024-12-04T19:16:02,262 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 48c104a4c7a51d8b62d084d2a093043a in 855ms, sequenceid=157, compaction requested=false 2024-12-04T19:16:02,262 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 48c104a4c7a51d8b62d084d2a093043a: 2024-12-04T19:16:02,311 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-04T19:16:02,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:16:02,311 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing 48c104a4c7a51d8b62d084d2a093043a 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-04T19:16:02,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 48c104a4c7a51d8b62d084d2a093043a, store=A 2024-12-04T19:16:02,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:16:02,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 48c104a4c7a51d8b62d084d2a093043a, store=B 2024-12-04T19:16:02,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:16:02,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 48c104a4c7a51d8b62d084d2a093043a, store=C 2024-12-04T19:16:02,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:16:02,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/A/41555f355f40422b8724c072fbdd035a is 50, key is test_row_0/A:col10/1733339761426/Put/seqid=0 2024-12-04T19:16:02,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742305_1481 (size=12151) 2024-12-04T19:16:02,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-04T19:16:02,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(8855): Flush requested on 48c104a4c7a51d8b62d084d2a093043a 2024-12-04T19:16:02,576 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. as already flushing 2024-12-04T19:16:02,608 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:02,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52496 deadline: 1733339822597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:02,609 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:02,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52588 deadline: 1733339822599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:02,609 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:02,609 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:02,609 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:02,609 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:02,609 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:02,609 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:02,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52494 deadline: 1733339822599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:02,609 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:02,609 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:02,609 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:02,609 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:02,610 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:02,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52544 deadline: 1733339822600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:02,610 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:02,610 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:02,610 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:02,610 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:02,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52546 deadline: 1733339822601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:02,610 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:02,611 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:02,611 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:02,715 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:02,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52496 deadline: 1733339822711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:02,715 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:02,715 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:02,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52588 deadline: 1733339822711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:02,716 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:02,716 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:02,716 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:02,716 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:02,716 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:02,716 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:02,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52494 deadline: 1733339822712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:02,716 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:02,716 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:02,716 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:02,716 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:02,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52546 deadline: 1733339822712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:02,716 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:02,716 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:02,716 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:02,719 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/A/41555f355f40422b8724c072fbdd035a 2024-12-04T19:16:02,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/B/08cbe18658a64160ba533074ce25a570 is 50, key is test_row_0/B:col10/1733339761426/Put/seqid=0 2024-12-04T19:16:02,729 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:02,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52544 deadline: 1733339822722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:02,731 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:02,731 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:02,731 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:02,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742306_1482 (size=12151) 2024-12-04T19:16:02,926 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:02,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52496 deadline: 1733339822921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:02,926 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:02,926 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:02,926 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:02,926 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:02,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52588 deadline: 1733339822921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:02,926 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:02,926 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:02,926 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:02,927 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:02,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52494 deadline: 1733339822921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:02,927 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:02,927 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:02,927 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:02,927 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:02,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52546 deadline: 1733339822922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:02,927 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:02,927 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:02,927 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:02,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:02,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52544 deadline: 1733339822942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:02,946 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:02,946 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:02,946 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:03,135 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/B/08cbe18658a64160ba533074ce25a570 2024-12-04T19:16:03,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/C/698f6244952648838ae12a6981722d8a is 50, key is test_row_0/C:col10/1733339761426/Put/seqid=0 2024-12-04T19:16:03,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742307_1483 (size=12151) 2024-12-04T19:16:03,236 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:03,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52496 deadline: 1733339823232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:03,237 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:03,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52494 deadline: 1733339823232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:03,237 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:03,237 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:03,237 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:03,237 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:03,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:03,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52588 deadline: 1733339823232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:03,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:03,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:03,237 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:03,237 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:03,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52546 deadline: 1733339823232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:03,237 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:03,237 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:03,238 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:03,238 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:03,238 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:03,256 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:03,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52544 deadline: 1733339823251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:03,257 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:03,257 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:03,257 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:03,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-04T19:16:03,545 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/C/698f6244952648838ae12a6981722d8a 2024-12-04T19:16:03,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/A/41555f355f40422b8724c072fbdd035a as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/41555f355f40422b8724c072fbdd035a 2024-12-04T19:16:03,552 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/41555f355f40422b8724c072fbdd035a, entries=150, sequenceid=170, filesize=11.9 K 2024-12-04T19:16:03,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/B/08cbe18658a64160ba533074ce25a570 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/08cbe18658a64160ba533074ce25a570 2024-12-04T19:16:03,555 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/08cbe18658a64160ba533074ce25a570, entries=150, sequenceid=170, filesize=11.9 K 2024-12-04T19:16:03,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/C/698f6244952648838ae12a6981722d8a as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/698f6244952648838ae12a6981722d8a 2024-12-04T19:16:03,560 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/698f6244952648838ae12a6981722d8a, entries=150, sequenceid=170, filesize=11.9 K 2024-12-04T19:16:03,561 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 48c104a4c7a51d8b62d084d2a093043a in 1250ms, sequenceid=170, compaction requested=true 2024-12-04T19:16:03,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for 48c104a4c7a51d8b62d084d2a093043a: 2024-12-04T19:16:03,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:16:03,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-12-04T19:16:03,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-12-04T19:16:03,564 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-12-04T19:16:03,564 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1670 sec 2024-12-04T19:16:03,565 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 2.1720 sec 2024-12-04T19:16:03,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(8855): Flush requested on 48c104a4c7a51d8b62d084d2a093043a 2024-12-04T19:16:03,751 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 48c104a4c7a51d8b62d084d2a093043a 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-04T19:16:03,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 48c104a4c7a51d8b62d084d2a093043a, store=A 2024-12-04T19:16:03,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T19:16:03,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 48c104a4c7a51d8b62d084d2a093043a, store=B 2024-12-04T19:16:03,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T19:16:03,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 48c104a4c7a51d8b62d084d2a093043a, store=C 2024-12-04T19:16:03,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T19:16:03,754 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/A/fc684417244845f7ae4bb5ed1a55d62f is 50, key is test_row_0/A:col10/1733339763749/Put/seqid=0 2024-12-04T19:16:03,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742308_1484 (size=16931) 2024-12-04T19:16:03,774 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:03,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52588 deadline: 1733339823764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:03,774 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:03,774 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:03,774 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:03,781 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:03,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52494 deadline: 1733339823771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:03,781 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:03,781 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:03,781 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:03,781 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:03,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52496 deadline: 1733339823772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:03,781 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:03,781 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:03,781 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:03,784 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:03,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52544 deadline: 1733339823774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:03,784 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:03,784 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:03,784 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:03,784 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:03,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52546 deadline: 1733339823774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:03,784 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:03,784 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:03,784 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:03,887 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:03,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52588 deadline: 1733339823881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:03,888 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:03,888 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:03,888 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:03,895 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:03,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52494 deadline: 1733339823892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:03,895 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:03,895 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:03,895 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:03,896 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:03,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52544 deadline: 1733339823892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:03,896 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:03,896 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:03,896 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:03,896 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:03,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52496 deadline: 1733339823892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:03,896 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:03,896 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:03,896 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:03,899 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:03,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52546 deadline: 1733339823892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:03,899 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:03,899 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:03,899 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:04,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:04,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52588 deadline: 1733339824091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:04,098 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:04,098 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:04,098 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:04,106 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:04,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52494 deadline: 1733339824101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:04,106 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:04,106 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:04,107 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:04,107 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:04,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52544 deadline: 1733339824102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:04,107 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:04,107 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:04,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52496 deadline: 1733339824102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:04,107 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:04,107 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:04,107 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:04,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52546 deadline: 1733339824102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:04,107 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:04,107 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:04,107 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:04,107 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:04,107 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:04,107 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:04,158 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/A/fc684417244845f7ae4bb5ed1a55d62f 2024-12-04T19:16:04,163 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/B/b4273718129a4229b2bb7235ca322c80 is 50, key is test_row_0/B:col10/1733339763749/Put/seqid=0 2024-12-04T19:16:04,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742309_1485 (size=12151) 2024-12-04T19:16:04,410 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:04,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52588 deadline: 1733339824402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:04,410 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:04,410 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:04,410 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:04,417 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:04,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52494 deadline: 1733339824411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:04,418 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:04,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52496 deadline: 1733339824411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:04,418 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:04,418 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:04,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52544 deadline: 1733339824412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:04,418 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:04,418 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:04,418 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:04,418 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:04,418 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:04,418 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:04,418 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:04,418 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:04,419 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:04,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52546 deadline: 1733339824412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:04,419 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:04,419 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:04,419 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:04,568 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/B/b4273718129a4229b2bb7235ca322c80 2024-12-04T19:16:04,579 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/C/abe3860b56d0448c8e2a2d0066720db1 is 50, key is test_row_0/C:col10/1733339763749/Put/seqid=0 2024-12-04T19:16:04,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742310_1486 (size=12151) 2024-12-04T19:16:04,614 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/C/abe3860b56d0448c8e2a2d0066720db1 2024-12-04T19:16:04,623 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/A/fc684417244845f7ae4bb5ed1a55d62f as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/fc684417244845f7ae4bb5ed1a55d62f 2024-12-04T19:16:04,631 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/fc684417244845f7ae4bb5ed1a55d62f, entries=250, sequenceid=191, filesize=16.5 K 2024-12-04T19:16:04,632 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/B/b4273718129a4229b2bb7235ca322c80 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/b4273718129a4229b2bb7235ca322c80 2024-12-04T19:16:04,637 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/b4273718129a4229b2bb7235ca322c80, entries=150, sequenceid=191, filesize=11.9 K 2024-12-04T19:16:04,637 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/C/abe3860b56d0448c8e2a2d0066720db1 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/abe3860b56d0448c8e2a2d0066720db1 2024-12-04T19:16:04,643 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/abe3860b56d0448c8e2a2d0066720db1, entries=150, sequenceid=191, filesize=11.9 K 2024-12-04T19:16:04,643 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 48c104a4c7a51d8b62d084d2a093043a in 893ms, sequenceid=191, compaction requested=true 2024-12-04T19:16:04,643 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 48c104a4c7a51d8b62d084d2a093043a: 2024-12-04T19:16:04,643 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 48c104a4c7a51d8b62d084d2a093043a:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T19:16:04,643 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:16:04,643 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 48c104a4c7a51d8b62d084d2a093043a:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T19:16:04,643 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T19:16:04,643 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:16:04,643 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T19:16:04,643 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 48c104a4c7a51d8b62d084d2a093043a:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T19:16:04,643 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T19:16:04,644 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48912 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T19:16:04,644 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1541): 48c104a4c7a51d8b62d084d2a093043a/B is initiating minor compaction (all files) 2024-12-04T19:16:04,645 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 48c104a4c7a51d8b62d084d2a093043a/B in TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:16:04,645 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/11a6907ef0c64f44b73d9a4f5f8b9b65, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/80252d0e2ca6455ba0189160b4ed543c, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/08cbe18658a64160ba533074ce25a570, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/b4273718129a4229b2bb7235ca322c80] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp, totalSize=47.8 K 2024-12-04T19:16:04,645 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 58472 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T19:16:04,645 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1541): 48c104a4c7a51d8b62d084d2a093043a/A is initiating minor compaction (all files) 2024-12-04T19:16:04,645 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 48c104a4c7a51d8b62d084d2a093043a/A in TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:16:04,645 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/df90f1337ba54599bd1c10c45803e2f7, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/db79c289b4854195b97f58a565038847, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/41555f355f40422b8724c072fbdd035a, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/fc684417244845f7ae4bb5ed1a55d62f] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp, totalSize=57.1 K 2024-12-04T19:16:04,645 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 11a6907ef0c64f44b73d9a4f5f8b9b65, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1733339759593 2024-12-04T19:16:04,645 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting df90f1337ba54599bd1c10c45803e2f7, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1733339759593 2024-12-04T19:16:04,645 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 80252d0e2ca6455ba0189160b4ed543c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1733339760248 2024-12-04T19:16:04,645 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting db79c289b4854195b97f58a565038847, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1733339760248 2024-12-04T19:16:04,646 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 08cbe18658a64160ba533074ce25a570, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1733339761418 2024-12-04T19:16:04,646 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 41555f355f40422b8724c072fbdd035a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1733339761418 2024-12-04T19:16:04,646 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting b4273718129a4229b2bb7235ca322c80, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1733339762598 2024-12-04T19:16:04,646 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting fc684417244845f7ae4bb5ed1a55d62f, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1733339762598 2024-12-04T19:16:04,670 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 48c104a4c7a51d8b62d084d2a093043a#A#compaction#410 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:16:04,670 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/A/e3afc2777545437aa91d15a793aac0dc is 50, key is test_row_0/A:col10/1733339763749/Put/seqid=0 2024-12-04T19:16:04,673 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 48c104a4c7a51d8b62d084d2a093043a#B#compaction#411 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:16:04,674 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/B/fd0eabb2bb13442f9674df7bf6d1894c is 50, key is test_row_0/B:col10/1733339763749/Put/seqid=0 2024-12-04T19:16:04,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742311_1487 (size=12595) 2024-12-04T19:16:04,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742312_1488 (size=12595) 2024-12-04T19:16:04,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(8855): Flush requested on 48c104a4c7a51d8b62d084d2a093043a 2024-12-04T19:16:04,924 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 48c104a4c7a51d8b62d084d2a093043a 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-04T19:16:04,924 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 48c104a4c7a51d8b62d084d2a093043a, store=A 2024-12-04T19:16:04,925 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:16:04,925 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 48c104a4c7a51d8b62d084d2a093043a, store=B 2024-12-04T19:16:04,925 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:16:04,925 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 48c104a4c7a51d8b62d084d2a093043a, store=C 2024-12-04T19:16:04,925 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:16:04,942 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/A/ab05112c0eb1477788bb02bee9ca6b0f is 50, key is test_row_0/A:col10/1733339764922/Put/seqid=0 2024-12-04T19:16:04,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742313_1489 (size=14541) 2024-12-04T19:16:04,981 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:04,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52546 deadline: 1733339824966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:04,982 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:04,982 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:04,982 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:04,982 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:04,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52588 deadline: 1733339824968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:04,982 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:04,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52544 deadline: 1733339824972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:04,983 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:04,983 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:04,983 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:04,983 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:04,983 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:04,983 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:04,988 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:04,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52494 deadline: 1733339824973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:04,989 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:04,989 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:04,989 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:04,997 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:04,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52496 deadline: 1733339824981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:04,997 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:04,997 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:04,997 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:05,095 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/A/e3afc2777545437aa91d15a793aac0dc as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/e3afc2777545437aa91d15a793aac0dc 2024-12-04T19:16:05,098 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/B/fd0eabb2bb13442f9674df7bf6d1894c as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/fd0eabb2bb13442f9674df7bf6d1894c 2024-12-04T19:16:05,103 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:05,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52546 deadline: 1733339825092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:05,103 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:05,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52588 deadline: 1733339825092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:05,103 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:05,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52544 deadline: 1733339825092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:05,104 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:05,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52494 deadline: 1733339825092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:05,104 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:05,104 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:05,104 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:05,104 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:05,104 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:05,104 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:05,104 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:05,104 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:05,104 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:05,104 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:05,104 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:05,104 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:05,105 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 48c104a4c7a51d8b62d084d2a093043a/A of 48c104a4c7a51d8b62d084d2a093043a into e3afc2777545437aa91d15a793aac0dc(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:16:05,105 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 48c104a4c7a51d8b62d084d2a093043a: 2024-12-04T19:16:05,105 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., storeName=48c104a4c7a51d8b62d084d2a093043a/A, priority=12, startTime=1733339764643; duration=0sec 2024-12-04T19:16:05,105 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T19:16:05,105 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 48c104a4c7a51d8b62d084d2a093043a:A 2024-12-04T19:16:05,105 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T19:16:05,105 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:05,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52496 deadline: 1733339825101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:05,106 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:05,106 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:05,106 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:05,107 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 48c104a4c7a51d8b62d084d2a093043a/B of 48c104a4c7a51d8b62d084d2a093043a into fd0eabb2bb13442f9674df7bf6d1894c(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:16:05,107 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 48c104a4c7a51d8b62d084d2a093043a: 2024-12-04T19:16:05,107 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., storeName=48c104a4c7a51d8b62d084d2a093043a/B, priority=12, startTime=1733339764643; duration=0sec 2024-12-04T19:16:05,107 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:16:05,107 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 48c104a4c7a51d8b62d084d2a093043a:B 2024-12-04T19:16:05,109 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48912 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T19:16:05,109 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1541): 48c104a4c7a51d8b62d084d2a093043a/C is initiating minor compaction (all files) 2024-12-04T19:16:05,109 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 48c104a4c7a51d8b62d084d2a093043a/C in TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:16:05,109 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/cda0c3296f8a4bf18b36f030e8a76ec1, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/baa05d747946429a8756ac1bb3126219, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/698f6244952648838ae12a6981722d8a, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/abe3860b56d0448c8e2a2d0066720db1] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp, totalSize=47.8 K 2024-12-04T19:16:05,109 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting cda0c3296f8a4bf18b36f030e8a76ec1, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1733339759593 2024-12-04T19:16:05,110 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting baa05d747946429a8756ac1bb3126219, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1733339760248 2024-12-04T19:16:05,110 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 698f6244952648838ae12a6981722d8a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1733339761418 2024-12-04T19:16:05,110 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting abe3860b56d0448c8e2a2d0066720db1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1733339762598 2024-12-04T19:16:05,130 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 48c104a4c7a51d8b62d084d2a093043a#C#compaction#413 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:16:05,130 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/C/bba9c182323a4afc9a919f4bcb72cafd is 50, key is test_row_0/C:col10/1733339763749/Put/seqid=0 2024-12-04T19:16:05,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742314_1490 (size=12595) 2024-12-04T19:16:05,156 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/C/bba9c182323a4afc9a919f4bcb72cafd as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/bba9c182323a4afc9a919f4bcb72cafd 2024-12-04T19:16:05,170 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 48c104a4c7a51d8b62d084d2a093043a/C of 48c104a4c7a51d8b62d084d2a093043a into bba9c182323a4afc9a919f4bcb72cafd(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:16:05,170 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 48c104a4c7a51d8b62d084d2a093043a: 2024-12-04T19:16:05,170 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., storeName=48c104a4c7a51d8b62d084d2a093043a/C, priority=12, startTime=1733339764643; duration=0sec 2024-12-04T19:16:05,170 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:16:05,170 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 48c104a4c7a51d8b62d084d2a093043a:C 2024-12-04T19:16:05,316 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:05,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52494 deadline: 1733339825312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:05,316 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:05,316 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:05,316 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:05,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52544 deadline: 1733339825312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:05,316 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:05,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:05,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52546 deadline: 1733339825312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:05,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:05,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52496 deadline: 1733339825312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:05,317 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:05,317 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:05,317 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:05,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:05,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52588 deadline: 1733339825312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:05,318 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:05,318 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:05,318 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:05,318 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:05,318 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:05,318 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:05,318 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:05,318 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:05,318 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:05,369 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/A/ab05112c0eb1477788bb02bee9ca6b0f 2024-12-04T19:16:05,389 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/B/19a41947da134437805880ea6a94665f is 50, key is test_row_0/B:col10/1733339764922/Put/seqid=0 2024-12-04T19:16:05,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742315_1491 (size=12151) 2024-12-04T19:16:05,426 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/B/19a41947da134437805880ea6a94665f 2024-12-04T19:16:05,439 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/C/f827d30274234dc594221ccb0a525e9d is 50, key is test_row_0/C:col10/1733339764922/Put/seqid=0 2024-12-04T19:16:05,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742316_1492 (size=12151) 2024-12-04T19:16:05,472 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/C/f827d30274234dc594221ccb0a525e9d 2024-12-04T19:16:05,481 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/A/ab05112c0eb1477788bb02bee9ca6b0f as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/ab05112c0eb1477788bb02bee9ca6b0f 2024-12-04T19:16:05,492 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/ab05112c0eb1477788bb02bee9ca6b0f, entries=200, sequenceid=203, filesize=14.2 K 2024-12-04T19:16:05,493 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/B/19a41947da134437805880ea6a94665f as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/19a41947da134437805880ea6a94665f 2024-12-04T19:16:05,499 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/19a41947da134437805880ea6a94665f, entries=150, sequenceid=203, filesize=11.9 K 2024-12-04T19:16:05,500 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/C/f827d30274234dc594221ccb0a525e9d as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/f827d30274234dc594221ccb0a525e9d 2024-12-04T19:16:05,503 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/f827d30274234dc594221ccb0a525e9d, entries=150, sequenceid=203, filesize=11.9 K 2024-12-04T19:16:05,504 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 48c104a4c7a51d8b62d084d2a093043a in 580ms, sequenceid=203, compaction requested=false 2024-12-04T19:16:05,504 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 48c104a4c7a51d8b62d084d2a093043a: 2024-12-04T19:16:05,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-04T19:16:05,552 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T19:16:05,552 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T19:16:05,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-12-04T19:16:05,554 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T19:16:05,555 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T19:16:05,555 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T19:16:05,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-04T19:16:05,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(8855): Flush requested on 48c104a4c7a51d8b62d084d2a093043a 2024-12-04T19:16:05,630 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 48c104a4c7a51d8b62d084d2a093043a 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-04T19:16:05,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 48c104a4c7a51d8b62d084d2a093043a, store=A 2024-12-04T19:16:05,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T19:16:05,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 48c104a4c7a51d8b62d084d2a093043a, store=B 2024-12-04T19:16:05,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T19:16:05,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 48c104a4c7a51d8b62d084d2a093043a, store=C 2024-12-04T19:16:05,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T19:16:05,638 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/A/e792f178edb64030abbe344ec3d6f3e0 is 50, key is test_row_0/A:col10/1733339764972/Put/seqid=0 2024-12-04T19:16:05,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742317_1493 (size=12151) 2024-12-04T19:16:05,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:05,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52544 deadline: 1733339825637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:05,652 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:05,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52546 deadline: 1733339825639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:05,652 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:05,652 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:05,652 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:05,653 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:05,653 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:05,653 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:05,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-04T19:16:05,664 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:05,664 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:05,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52496 deadline: 1733339825651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:05,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52494 deadline: 1733339825649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:05,664 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:05,664 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:05,664 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:05,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52588 deadline: 1733339825652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:05,664 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:05,664 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:05,664 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:05,664 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:05,665 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:05,665 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:05,665 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:05,707 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-04T19:16:05,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:16:05,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. as already flushing 2024-12-04T19:16:05,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:16:05,708 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:05,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:05,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:05,766 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:05,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52544 deadline: 1733339825761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:05,766 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:05,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52546 deadline: 1733339825761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:05,767 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:05,767 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:05,767 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:05,767 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:05,767 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:05,767 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:05,775 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:05,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52496 deadline: 1733339825772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:05,775 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:05,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52588 deadline: 1733339825772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:05,775 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:05,775 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:05,775 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:05,775 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:05,775 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:05,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52494 deadline: 1733339825772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:05,775 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:05,775 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:05,776 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:05,776 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:05,776 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:05,860 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-04T19:16:05,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:16:05,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. as already flushing 2024-12-04T19:16:05,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:16:05,861 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:05,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:05,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:05,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-04T19:16:05,976 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:05,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52544 deadline: 1733339825971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:05,976 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:05,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52546 deadline: 1733339825971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:05,976 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:05,976 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:05,976 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:05,976 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:05,976 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:05,976 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:05,987 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:05,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52496 deadline: 1733339825981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:05,987 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:05,987 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:05,987 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:05,987 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:05,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52588 deadline: 1733339825982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:05,987 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:05,987 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:05,987 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:05,987 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:05,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52494 deadline: 1733339825982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:05,988 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:05,988 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:05,988 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:06,013 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-04T19:16:06,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:16:06,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. as already flushing 2024-12-04T19:16:06,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:16:06,013 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:06,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:06,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:06,046 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/A/e792f178edb64030abbe344ec3d6f3e0 2024-12-04T19:16:06,052 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/B/d2350030940b4135bce171c03373bb27 is 50, key is test_row_0/B:col10/1733339764972/Put/seqid=0 2024-12-04T19:16:06,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742318_1494 (size=12151) 2024-12-04T19:16:06,166 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-04T19:16:06,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:16:06,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. as already flushing 2024-12-04T19:16:06,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:16:06,166 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:06,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:06,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:06,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-04T19:16:06,288 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:06,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52544 deadline: 1733339826284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:06,289 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:06,288 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:06,289 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:06,289 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:06,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52546 deadline: 1733339826284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:06,289 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:06,289 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:06,289 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:06,296 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:06,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52496 deadline: 1733339826291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:06,296 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:06,296 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:06,296 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:06,296 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:06,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52494 deadline: 1733339826292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:06,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:06,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:06,296 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:06,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:06,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52588 deadline: 1733339826292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:06,297 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:06,297 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:06,297 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:06,317 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-04T19:16:06,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:16:06,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. as already flushing 2024-12-04T19:16:06,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:16:06,317 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:06,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:06,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:06,460 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/B/d2350030940b4135bce171c03373bb27 2024-12-04T19:16:06,465 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/C/6204e0fa677c446c88d2c9674c024674 is 50, key is test_row_0/C:col10/1733339764972/Put/seqid=0 2024-12-04T19:16:06,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742319_1495 (size=12151) 2024-12-04T19:16:06,470 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-04T19:16:06,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:16:06,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. as already flushing 2024-12-04T19:16:06,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:16:06,470 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:06,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:06,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:06,622 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-04T19:16:06,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:16:06,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. as already flushing 2024-12-04T19:16:06,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:16:06,623 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:06,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:06,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:06,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-04T19:16:06,775 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-04T19:16:06,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:16:06,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. as already flushing 2024-12-04T19:16:06,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:16:06,775 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:06,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:06,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:06,806 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:06,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52544 deadline: 1733339826801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:06,806 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:06,806 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:06,806 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:06,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52496 deadline: 1733339826802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:06,806 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:06,806 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:06,806 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:06,806 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:06,807 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:06,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52494 deadline: 1733339826802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:06,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:06,807 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:06,807 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:06,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52546 deadline: 1733339826802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:06,807 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:06,807 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:06,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:06,807 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:06,807 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:06,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52588 deadline: 1733339826802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:06,807 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:06,807 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:06,807 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:06,869 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/C/6204e0fa677c446c88d2c9674c024674 2024-12-04T19:16:06,872 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/A/e792f178edb64030abbe344ec3d6f3e0 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/e792f178edb64030abbe344ec3d6f3e0 2024-12-04T19:16:06,875 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/e792f178edb64030abbe344ec3d6f3e0, entries=150, sequenceid=227, filesize=11.9 K 2024-12-04T19:16:06,875 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/B/d2350030940b4135bce171c03373bb27 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/d2350030940b4135bce171c03373bb27 2024-12-04T19:16:06,878 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/d2350030940b4135bce171c03373bb27, entries=150, sequenceid=227, filesize=11.9 K 2024-12-04T19:16:06,879 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/C/6204e0fa677c446c88d2c9674c024674 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/6204e0fa677c446c88d2c9674c024674 2024-12-04T19:16:06,881 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/6204e0fa677c446c88d2c9674c024674, entries=150, sequenceid=227, filesize=11.9 K 2024-12-04T19:16:06,882 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 48c104a4c7a51d8b62d084d2a093043a in 1252ms, sequenceid=227, compaction requested=true 2024-12-04T19:16:06,882 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 48c104a4c7a51d8b62d084d2a093043a: 2024-12-04T19:16:06,882 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 48c104a4c7a51d8b62d084d2a093043a:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T19:16:06,882 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:16:06,882 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T19:16:06,882 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 48c104a4c7a51d8b62d084d2a093043a:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T19:16:06,882 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T19:16:06,882 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 48c104a4c7a51d8b62d084d2a093043a:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T19:16:06,882 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-04T19:16:06,883 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T19:16:06,883 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T19:16:06,883 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39287 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T19:16:06,883 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1541): 48c104a4c7a51d8b62d084d2a093043a/B is initiating minor compaction (all files) 2024-12-04T19:16:06,883 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1541): 48c104a4c7a51d8b62d084d2a093043a/A is initiating minor compaction (all files) 2024-12-04T19:16:06,883 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 48c104a4c7a51d8b62d084d2a093043a/B in TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:16:06,883 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 48c104a4c7a51d8b62d084d2a093043a/A in TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:16:06,883 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/e3afc2777545437aa91d15a793aac0dc, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/ab05112c0eb1477788bb02bee9ca6b0f, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/e792f178edb64030abbe344ec3d6f3e0] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp, totalSize=38.4 K 2024-12-04T19:16:06,883 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/fd0eabb2bb13442f9674df7bf6d1894c, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/19a41947da134437805880ea6a94665f, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/d2350030940b4135bce171c03373bb27] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp, totalSize=36.0 K 2024-12-04T19:16:06,884 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting fd0eabb2bb13442f9674df7bf6d1894c, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1733339762598 2024-12-04T19:16:06,884 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting e3afc2777545437aa91d15a793aac0dc, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1733339762598 2024-12-04T19:16:06,884 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting ab05112c0eb1477788bb02bee9ca6b0f, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1733339763760 2024-12-04T19:16:06,884 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 19a41947da134437805880ea6a94665f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1733339763760 2024-12-04T19:16:06,884 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting d2350030940b4135bce171c03373bb27, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1733339764972 2024-12-04T19:16:06,884 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting e792f178edb64030abbe344ec3d6f3e0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1733339764972 2024-12-04T19:16:06,889 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 48c104a4c7a51d8b62d084d2a093043a#A#compaction#419 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:16:06,890 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/A/f4ebbc78104f4115a1cc65303850076c is 50, key is test_row_0/A:col10/1733339764972/Put/seqid=0 2024-12-04T19:16:06,890 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 48c104a4c7a51d8b62d084d2a093043a#B#compaction#420 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:16:06,890 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/B/e4ee6cb89cde49039f135729b6a8e4bd is 50, key is test_row_0/B:col10/1733339764972/Put/seqid=0 2024-12-04T19:16:06,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742320_1496 (size=12697) 2024-12-04T19:16:06,899 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/A/f4ebbc78104f4115a1cc65303850076c as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/f4ebbc78104f4115a1cc65303850076c 2024-12-04T19:16:06,905 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 48c104a4c7a51d8b62d084d2a093043a/A of 48c104a4c7a51d8b62d084d2a093043a into f4ebbc78104f4115a1cc65303850076c(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:16:06,905 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 48c104a4c7a51d8b62d084d2a093043a: 2024-12-04T19:16:06,905 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., storeName=48c104a4c7a51d8b62d084d2a093043a/A, priority=13, startTime=1733339766882; duration=0sec 2024-12-04T19:16:06,905 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T19:16:06,905 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 48c104a4c7a51d8b62d084d2a093043a:A 2024-12-04T19:16:06,905 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T19:16:06,908 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T19:16:06,908 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1541): 48c104a4c7a51d8b62d084d2a093043a/C is initiating minor compaction (all files) 2024-12-04T19:16:06,908 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 48c104a4c7a51d8b62d084d2a093043a/C in TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:16:06,908 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/bba9c182323a4afc9a919f4bcb72cafd, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/f827d30274234dc594221ccb0a525e9d, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/6204e0fa677c446c88d2c9674c024674] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp, totalSize=36.0 K 2024-12-04T19:16:06,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742321_1497 (size=12697) 2024-12-04T19:16:06,912 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting bba9c182323a4afc9a919f4bcb72cafd, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1733339762598 2024-12-04T19:16:06,913 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting f827d30274234dc594221ccb0a525e9d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1733339763760 2024-12-04T19:16:06,914 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6204e0fa677c446c88d2c9674c024674, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1733339764972 2024-12-04T19:16:06,917 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/B/e4ee6cb89cde49039f135729b6a8e4bd as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/e4ee6cb89cde49039f135729b6a8e4bd 2024-12-04T19:16:06,924 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 48c104a4c7a51d8b62d084d2a093043a/B of 48c104a4c7a51d8b62d084d2a093043a into e4ee6cb89cde49039f135729b6a8e4bd(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:16:06,924 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 48c104a4c7a51d8b62d084d2a093043a: 2024-12-04T19:16:06,924 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., storeName=48c104a4c7a51d8b62d084d2a093043a/B, priority=13, startTime=1733339766882; duration=0sec 2024-12-04T19:16:06,924 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 48c104a4c7a51d8b62d084d2a093043a#C#compaction#421 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:16:06,924 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:16:06,924 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 48c104a4c7a51d8b62d084d2a093043a:B 2024-12-04T19:16:06,925 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/C/a4be3e0c72464e8292ac07b298300bcd is 50, key is test_row_0/C:col10/1733339764972/Put/seqid=0 2024-12-04T19:16:06,928 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-04T19:16:06,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:16:06,929 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing 48c104a4c7a51d8b62d084d2a093043a 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-04T19:16:06,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 48c104a4c7a51d8b62d084d2a093043a, store=A 2024-12-04T19:16:06,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:16:06,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 48c104a4c7a51d8b62d084d2a093043a, store=B 2024-12-04T19:16:06,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:16:06,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 48c104a4c7a51d8b62d084d2a093043a, store=C 2024-12-04T19:16:06,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:16:06,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/A/104dd64c7ec24a33bb3986099affa060 is 50, key is test_row_0/A:col10/1733339765650/Put/seqid=0 2024-12-04T19:16:06,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742322_1498 (size=12697) 2024-12-04T19:16:06,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742323_1499 (size=12151) 2024-12-04T19:16:06,955 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/C/a4be3e0c72464e8292ac07b298300bcd as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/a4be3e0c72464e8292ac07b298300bcd 2024-12-04T19:16:06,962 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 48c104a4c7a51d8b62d084d2a093043a/C of 48c104a4c7a51d8b62d084d2a093043a into a4be3e0c72464e8292ac07b298300bcd(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:16:06,962 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 48c104a4c7a51d8b62d084d2a093043a: 2024-12-04T19:16:06,962 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., storeName=48c104a4c7a51d8b62d084d2a093043a/C, priority=13, startTime=1733339766882; duration=0sec 2024-12-04T19:16:06,962 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:16:06,962 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 48c104a4c7a51d8b62d084d2a093043a:C 2024-12-04T19:16:07,350 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/A/104dd64c7ec24a33bb3986099affa060 2024-12-04T19:16:07,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/B/e5713631593b4afebea6af01da58e0de is 50, key is test_row_0/B:col10/1733339765650/Put/seqid=0 2024-12-04T19:16:07,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742324_1500 (size=12151) 2024-12-04T19:16:07,377 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/B/e5713631593b4afebea6af01da58e0de 2024-12-04T19:16:07,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/C/605ed11b9c484897b6e7f5a3d4537f65 is 50, key is test_row_0/C:col10/1733339765650/Put/seqid=0 2024-12-04T19:16:07,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742325_1501 (size=12151) 2024-12-04T19:16:07,399 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/C/605ed11b9c484897b6e7f5a3d4537f65 2024-12-04T19:16:07,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/A/104dd64c7ec24a33bb3986099affa060 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/104dd64c7ec24a33bb3986099affa060 2024-12-04T19:16:07,410 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/104dd64c7ec24a33bb3986099affa060, entries=150, sequenceid=240, filesize=11.9 K 2024-12-04T19:16:07,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/B/e5713631593b4afebea6af01da58e0de as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/e5713631593b4afebea6af01da58e0de 2024-12-04T19:16:07,416 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/e5713631593b4afebea6af01da58e0de, entries=150, sequenceid=240, filesize=11.9 K 2024-12-04T19:16:07,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/C/605ed11b9c484897b6e7f5a3d4537f65 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/605ed11b9c484897b6e7f5a3d4537f65 2024-12-04T19:16:07,423 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/605ed11b9c484897b6e7f5a3d4537f65, entries=150, sequenceid=240, filesize=11.9 K 2024-12-04T19:16:07,424 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=0 B/0 for 48c104a4c7a51d8b62d084d2a093043a in 496ms, sequenceid=240, compaction requested=false 2024-12-04T19:16:07,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for 48c104a4c7a51d8b62d084d2a093043a: 2024-12-04T19:16:07,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:16:07,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-12-04T19:16:07,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-12-04T19:16:07,432 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-12-04T19:16:07,432 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8740 sec 2024-12-04T19:16:07,433 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 1.8790 sec 2024-12-04T19:16:07,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-04T19:16:07,702 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T19:16:07,703 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T19:16:07,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-12-04T19:16:07,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-04T19:16:07,704 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T19:16:07,705 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T19:16:07,705 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T19:16:07,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-04T19:16:07,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(8855): Flush requested on 48c104a4c7a51d8b62d084d2a093043a 2024-12-04T19:16:07,820 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 48c104a4c7a51d8b62d084d2a093043a 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-04T19:16:07,820 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 48c104a4c7a51d8b62d084d2a093043a, store=A 2024-12-04T19:16:07,820 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:16:07,820 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 48c104a4c7a51d8b62d084d2a093043a, store=B 2024-12-04T19:16:07,820 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:16:07,820 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 48c104a4c7a51d8b62d084d2a093043a, store=C 2024-12-04T19:16:07,820 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:16:07,823 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/A/60dc8d2693a84c97b9e670b0779beecc is 50, key is test_row_0/A:col10/1733339767819/Put/seqid=0 2024-12-04T19:16:07,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742326_1502 (size=12147) 2024-12-04T19:16:07,846 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1733339767832, max timestamp=1733339767844Num uniques 859; 2024-12-04T19:16:07,846 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-12-04T19:16:07,847 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1733339767832, max timestamp=1733339767844Num uniques 859; 2024-12-04T19:16:07,847 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-12-04T19:16:07,847 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1733339767832, max timestamp=1733339767844Num uniques 859; 2024-12-04T19:16:07,847 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-12-04T19:16:07,856 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-04T19:16:07,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:16:07,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. as already flushing 2024-12-04T19:16:07,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:16:07,857 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:07,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:07,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:07,867 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:07,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52494 deadline: 1733339827864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:07,868 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:07,868 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:07,868 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:07,872 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:07,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52544 deadline: 1733339827865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:07,872 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:07,873 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:07,873 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:07,875 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:07,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52496 deadline: 1733339827866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:07,875 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:07,875 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:07,875 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:07,878 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:07,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52588 deadline: 1733339827867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:07,878 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:07,878 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:07,878 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:07,878 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:07,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52546 deadline: 1733339827867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:07,878 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:07,878 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:07,878 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:07,977 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:07,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52494 deadline: 1733339827972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:07,977 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:07,977 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:07,977 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:07,988 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:07,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52544 deadline: 1733339827982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:07,988 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:07,988 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:07,988 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:07,988 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:07,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52496 deadline: 1733339827982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:07,989 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:07,989 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:07,989 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:07,989 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:07,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52588 deadline: 1733339827982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:07,989 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:07,989 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:07,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52546 deadline: 1733339827982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:07,989 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:07,989 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:07,989 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:07,989 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:07,989 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:08,009 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-04T19:16:08,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:16:08,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. as already flushing 2024-12-04T19:16:08,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:16:08,010 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:08,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:08,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:08,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-04T19:16:08,162 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-04T19:16:08,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:16:08,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. as already flushing 2024-12-04T19:16:08,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:16:08,163 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:08,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:08,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:08,191 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:08,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52494 deadline: 1733339828182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:08,191 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:08,191 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:08,191 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:08,203 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:08,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52544 deadline: 1733339828192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:08,203 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:08,203 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:08,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52496 deadline: 1733339828192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:08,204 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:08,204 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:08,204 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:08,204 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:08,204 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:08,204 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:08,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52588 deadline: 1733339828192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:08,204 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:08,204 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:08,204 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:08,204 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:08,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52546 deadline: 1733339828192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:08,204 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:08,204 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:08,204 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:08,231 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/A/60dc8d2693a84c97b9e670b0779beecc 2024-12-04T19:16:08,236 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/B/e22741162ffd40ccbc4e1a818baf754c is 50, key is test_row_0/B:col10/1733339767819/Put/seqid=0 2024-12-04T19:16:08,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742327_1503 (size=9757) 2024-12-04T19:16:08,241 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/B/e22741162ffd40ccbc4e1a818baf754c 2024-12-04T19:16:08,246 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/C/755633419de647048543c9f4bce94ab5 is 50, key is test_row_0/C:col10/1733339767819/Put/seqid=0 2024-12-04T19:16:08,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742328_1504 (size=9757) 2024-12-04T19:16:08,315 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-04T19:16:08,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:16:08,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. as already flushing 2024-12-04T19:16:08,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:16:08,315 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:08,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:08,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:08,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-04T19:16:08,467 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-04T19:16:08,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:16:08,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. as already flushing 2024-12-04T19:16:08,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:16:08,468 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:08,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:08,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:08,507 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:08,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52494 deadline: 1733339828502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:08,508 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:08,508 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:08,508 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:08,521 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:08,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52544 deadline: 1733339828512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:08,522 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:08,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52496 deadline: 1733339828512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:08,522 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:08,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52546 deadline: 1733339828512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:08,522 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:08,522 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:08,523 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:08,523 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:08,523 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:08,523 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:08,523 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:08,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52588 deadline: 1733339828512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:08,528 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:08,528 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:08,528 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:08,528 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:08,528 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:08,528 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:08,620 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-04T19:16:08,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:16:08,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. as already flushing 2024-12-04T19:16:08,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:16:08,620 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:08,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:08,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:08,674 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/C/755633419de647048543c9f4bce94ab5 2024-12-04T19:16:08,681 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/A/60dc8d2693a84c97b9e670b0779beecc as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/60dc8d2693a84c97b9e670b0779beecc 2024-12-04T19:16:08,686 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/60dc8d2693a84c97b9e670b0779beecc, entries=150, sequenceid=251, filesize=11.9 K 2024-12-04T19:16:08,686 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/B/e22741162ffd40ccbc4e1a818baf754c as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/e22741162ffd40ccbc4e1a818baf754c 2024-12-04T19:16:08,689 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/e22741162ffd40ccbc4e1a818baf754c, entries=100, sequenceid=251, filesize=9.5 K 2024-12-04T19:16:08,689 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/C/755633419de647048543c9f4bce94ab5 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/755633419de647048543c9f4bce94ab5 2024-12-04T19:16:08,692 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/755633419de647048543c9f4bce94ab5, entries=100, sequenceid=251, filesize=9.5 K 2024-12-04T19:16:08,692 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for 48c104a4c7a51d8b62d084d2a093043a in 872ms, sequenceid=251, compaction requested=true 2024-12-04T19:16:08,692 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 48c104a4c7a51d8b62d084d2a093043a: 2024-12-04T19:16:08,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 48c104a4c7a51d8b62d084d2a093043a:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T19:16:08,693 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:16:08,693 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T19:16:08,693 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 48c104a4c7a51d8b62d084d2a093043a:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T19:16:08,693 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:16:08,693 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 48c104a4c7a51d8b62d084d2a093043a:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T19:16:08,693 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T19:16:08,693 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T19:16:08,694 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36995 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T19:16:08,694 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34605 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T19:16:08,694 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1541): 48c104a4c7a51d8b62d084d2a093043a/B is initiating minor compaction (all files) 2024-12-04T19:16:08,694 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1541): 48c104a4c7a51d8b62d084d2a093043a/A is initiating minor compaction (all files) 2024-12-04T19:16:08,694 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 48c104a4c7a51d8b62d084d2a093043a/A in TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:16:08,694 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 48c104a4c7a51d8b62d084d2a093043a/B in TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:16:08,694 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/e4ee6cb89cde49039f135729b6a8e4bd, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/e5713631593b4afebea6af01da58e0de, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/e22741162ffd40ccbc4e1a818baf754c] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp, totalSize=33.8 K 2024-12-04T19:16:08,694 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/f4ebbc78104f4115a1cc65303850076c, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/104dd64c7ec24a33bb3986099affa060, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/60dc8d2693a84c97b9e670b0779beecc] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp, totalSize=36.1 K 2024-12-04T19:16:08,694 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting f4ebbc78104f4115a1cc65303850076c, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1733339764972 2024-12-04T19:16:08,694 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting e4ee6cb89cde49039f135729b6a8e4bd, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1733339764972 2024-12-04T19:16:08,695 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting e5713631593b4afebea6af01da58e0de, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1733339765637 2024-12-04T19:16:08,695 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 104dd64c7ec24a33bb3986099affa060, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1733339765637 2024-12-04T19:16:08,695 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 60dc8d2693a84c97b9e670b0779beecc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1733339767814 2024-12-04T19:16:08,695 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting e22741162ffd40ccbc4e1a818baf754c, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1733339767814 2024-12-04T19:16:08,706 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 48c104a4c7a51d8b62d084d2a093043a#B#compaction#428 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:16:08,707 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/B/9ebae3da0cdd4c85bf191c43c98ac656 is 50, key is test_row_0/B:col10/1733339767819/Put/seqid=0 2024-12-04T19:16:08,707 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 48c104a4c7a51d8b62d084d2a093043a#A#compaction#429 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:16:08,707 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/A/ab9dd125c77b47539ff2afa30cbeedda is 50, key is test_row_0/A:col10/1733339767819/Put/seqid=0 2024-12-04T19:16:08,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742329_1505 (size=12799) 2024-12-04T19:16:08,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742330_1506 (size=12799) 2024-12-04T19:16:08,773 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-04T19:16:08,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:16:08,773 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing 48c104a4c7a51d8b62d084d2a093043a 3/3 column families, dataSize=140.89 KB heapSize=381.47 KB 2024-12-04T19:16:08,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 48c104a4c7a51d8b62d084d2a093043a, store=A 2024-12-04T19:16:08,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T19:16:08,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 48c104a4c7a51d8b62d084d2a093043a, store=B 2024-12-04T19:16:08,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T19:16:08,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 48c104a4c7a51d8b62d084d2a093043a, store=C 2024-12-04T19:16:08,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T19:16:08,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/A/282a125926eb423e82beabaf19ba1b47 is 50, key is test_row_0/A:col10/1733339767864/Put/seqid=0 2024-12-04T19:16:08,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742331_1507 (size=12301) 2024-12-04T19:16:08,784 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/A/282a125926eb423e82beabaf19ba1b47 2024-12-04T19:16:08,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/B/8e390ba636864a1f946b620f3b447715 is 50, key is test_row_0/B:col10/1733339767864/Put/seqid=0 2024-12-04T19:16:08,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742332_1508 (size=12301) 2024-12-04T19:16:08,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-04T19:16:09,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(8855): Flush requested on 48c104a4c7a51d8b62d084d2a093043a 2024-12-04T19:16:09,018 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. as already flushing 2024-12-04T19:16:09,040 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:09,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52494 deadline: 1733339829035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:09,041 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:09,041 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:09,041 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:09,044 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:09,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52546 deadline: 1733339829038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:09,044 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:09,044 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:09,044 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:09,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52496 deadline: 1733339829039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:09,044 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:09,044 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:09,044 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:09,044 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:09,049 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:09,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52588 deadline: 1733339829040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:09,049 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:09,049 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:09,049 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:09,049 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:09,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52544 deadline: 1733339829041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:09,049 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:09,050 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:09,050 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:09,130 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/A/ab9dd125c77b47539ff2afa30cbeedda as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/ab9dd125c77b47539ff2afa30cbeedda 2024-12-04T19:16:09,133 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/B/9ebae3da0cdd4c85bf191c43c98ac656 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/9ebae3da0cdd4c85bf191c43c98ac656 2024-12-04T19:16:09,134 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 48c104a4c7a51d8b62d084d2a093043a/A of 48c104a4c7a51d8b62d084d2a093043a into ab9dd125c77b47539ff2afa30cbeedda(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:16:09,134 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 48c104a4c7a51d8b62d084d2a093043a: 2024-12-04T19:16:09,134 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., storeName=48c104a4c7a51d8b62d084d2a093043a/A, priority=13, startTime=1733339768692; duration=0sec 2024-12-04T19:16:09,134 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T19:16:09,134 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 48c104a4c7a51d8b62d084d2a093043a:A 2024-12-04T19:16:09,134 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T19:16:09,135 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34605 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T19:16:09,136 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1541): 48c104a4c7a51d8b62d084d2a093043a/C is initiating minor compaction (all files) 2024-12-04T19:16:09,136 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 48c104a4c7a51d8b62d084d2a093043a/C in TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:16:09,136 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/a4be3e0c72464e8292ac07b298300bcd, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/605ed11b9c484897b6e7f5a3d4537f65, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/755633419de647048543c9f4bce94ab5] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp, totalSize=33.8 K 2024-12-04T19:16:09,136 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting a4be3e0c72464e8292ac07b298300bcd, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1733339764972 2024-12-04T19:16:09,136 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 605ed11b9c484897b6e7f5a3d4537f65, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1733339765637 2024-12-04T19:16:09,137 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 48c104a4c7a51d8b62d084d2a093043a/B of 48c104a4c7a51d8b62d084d2a093043a into 9ebae3da0cdd4c85bf191c43c98ac656(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:16:09,137 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 48c104a4c7a51d8b62d084d2a093043a: 2024-12-04T19:16:09,137 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., storeName=48c104a4c7a51d8b62d084d2a093043a/B, priority=13, startTime=1733339768693; duration=0sec 2024-12-04T19:16:09,137 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 755633419de647048543c9f4bce94ab5, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1733339767814 2024-12-04T19:16:09,137 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:16:09,137 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 48c104a4c7a51d8b62d084d2a093043a:B 2024-12-04T19:16:09,143 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 48c104a4c7a51d8b62d084d2a093043a#C#compaction#432 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:16:09,143 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/C/b716959b1102447685288f4963b327cc is 50, key is test_row_0/C:col10/1733339767819/Put/seqid=0 2024-12-04T19:16:09,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742333_1509 (size=12799) 2024-12-04T19:16:09,155 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:09,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52546 deadline: 1733339829152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:09,156 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:09,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52496 deadline: 1733339829152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:09,156 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:09,156 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:09,156 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:09,156 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:09,156 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:09,156 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:09,156 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:09,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52588 deadline: 1733339829152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:09,156 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:09,156 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:09,156 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:09,156 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:09,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52494 deadline: 1733339829152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:09,157 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:09,157 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:09,157 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:09,157 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:09,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52544 deadline: 1733339829152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:09,157 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:09,157 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:09,157 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:09,196 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/B/8e390ba636864a1f946b620f3b447715 2024-12-04T19:16:09,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/C/20010184066c47aba5acd7458944fb85 is 50, key is test_row_0/C:col10/1733339767864/Put/seqid=0 2024-12-04T19:16:09,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742334_1510 (size=12301) 2024-12-04T19:16:09,207 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/C/20010184066c47aba5acd7458944fb85 2024-12-04T19:16:09,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/A/282a125926eb423e82beabaf19ba1b47 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/282a125926eb423e82beabaf19ba1b47 2024-12-04T19:16:09,212 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/282a125926eb423e82beabaf19ba1b47, entries=150, sequenceid=284, filesize=12.0 K 2024-12-04T19:16:09,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/B/8e390ba636864a1f946b620f3b447715 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/8e390ba636864a1f946b620f3b447715 2024-12-04T19:16:09,220 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/8e390ba636864a1f946b620f3b447715, entries=150, sequenceid=284, filesize=12.0 K 2024-12-04T19:16:09,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/C/20010184066c47aba5acd7458944fb85 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/20010184066c47aba5acd7458944fb85 2024-12-04T19:16:09,226 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/20010184066c47aba5acd7458944fb85, entries=150, sequenceid=284, filesize=12.0 K 2024-12-04T19:16:09,227 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~381.42 KB/390576, currentSize=46.96 KB/48090 for 48c104a4c7a51d8b62d084d2a093043a in 454ms, sequenceid=284, compaction requested=false 2024-12-04T19:16:09,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for 48c104a4c7a51d8b62d084d2a093043a: 2024-12-04T19:16:09,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:16:09,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-12-04T19:16:09,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-12-04T19:16:09,231 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-12-04T19:16:09,231 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5230 sec 2024-12-04T19:16:09,232 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 1.5280 sec 2024-12-04T19:16:09,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(8855): Flush requested on 48c104a4c7a51d8b62d084d2a093043a 2024-12-04T19:16:09,365 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 48c104a4c7a51d8b62d084d2a093043a 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-04T19:16:09,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 48c104a4c7a51d8b62d084d2a093043a, store=A 2024-12-04T19:16:09,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:16:09,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 48c104a4c7a51d8b62d084d2a093043a, store=B 2024-12-04T19:16:09,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:16:09,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 48c104a4c7a51d8b62d084d2a093043a, store=C 2024-12-04T19:16:09,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:16:09,369 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/A/165cafd9b2c649ad9116015865fc4164 is 50, key is test_row_0/A:col10/1733339769038/Put/seqid=0 2024-12-04T19:16:09,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742335_1511 (size=14741) 2024-12-04T19:16:09,419 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1733339769406, max timestamp=1733339769418Num uniques 859; 2024-12-04T19:16:09,420 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-12-04T19:16:09,420 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1733339769406, max timestamp=1733339769418Num uniques 859; 2024-12-04T19:16:09,420 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-12-04T19:16:09,420 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1733339769406, max timestamp=1733339769418Num uniques 859; 2024-12-04T19:16:09,420 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-12-04T19:16:09,449 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:09,449 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:09,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52496 deadline: 1733339829439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:09,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52588 deadline: 1733339829441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:09,449 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:09,449 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:09,449 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:09,449 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:09,449 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:09,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52494 deadline: 1733339829448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:09,449 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:09,449 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:09,450 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:09,450 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:09,450 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:09,458 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:09,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52546 deadline: 1733339829449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:09,458 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:09,458 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:09,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52544 deadline: 1733339829449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:09,458 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:09,458 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:09,459 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:09,459 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:09,459 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:09,550 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/C/b716959b1102447685288f4963b327cc as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/b716959b1102447685288f4963b327cc 2024-12-04T19:16:09,554 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 48c104a4c7a51d8b62d084d2a093043a/C of 48c104a4c7a51d8b62d084d2a093043a into b716959b1102447685288f4963b327cc(size=12.5 K), total size for store is 24.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:16:09,554 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 48c104a4c7a51d8b62d084d2a093043a: 2024-12-04T19:16:09,554 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., storeName=48c104a4c7a51d8b62d084d2a093043a/C, priority=13, startTime=1733339768693; duration=0sec 2024-12-04T19:16:09,554 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:16:09,554 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 48c104a4c7a51d8b62d084d2a093043a:C 2024-12-04T19:16:09,556 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:09,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52496 deadline: 1733339829552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:09,556 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:09,556 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:09,556 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:09,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52588 deadline: 1733339829552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:09,556 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:09,557 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:09,557 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:09,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52494 deadline: 1733339829552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:09,557 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:09,557 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:09,557 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:09,557 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:09,557 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:09,570 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:09,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52546 deadline: 1733339829561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:09,571 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:09,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52544 deadline: 1733339829561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:09,571 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:09,571 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:09,571 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:09,571 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:09,571 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:09,571 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:09,769 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:09,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52496 deadline: 1733339829762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:09,769 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:09,769 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:09,769 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:09,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52588 deadline: 1733339829762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:09,769 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:09,769 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:09,769 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:09,769 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:09,769 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:09,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52494 deadline: 1733339829762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:09,770 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:09,770 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:09,770 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:09,782 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/A/165cafd9b2c649ad9116015865fc4164 2024-12-04T19:16:09,787 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/B/5cd3638558d94d949c9f0420362bd353 is 50, key is test_row_0/B:col10/1733339769038/Put/seqid=0 2024-12-04T19:16:09,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:09,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52546 deadline: 1733339829781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:09,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:09,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52544 deadline: 1733339829782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:09,790 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:09,791 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:09,791 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:09,792 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:09,792 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:09,792 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:09,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742336_1512 (size=12301) 2024-12-04T19:16:09,794 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/B/5cd3638558d94d949c9f0420362bd353 2024-12-04T19:16:09,799 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/C/4d98aac7ba314f2d9d4e397ee7d821c8 is 50, key is test_row_0/C:col10/1733339769038/Put/seqid=0 2024-12-04T19:16:09,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742337_1513 (size=12301) 2024-12-04T19:16:09,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-04T19:16:09,862 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T19:16:09,862 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T19:16:09,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-12-04T19:16:09,864 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T19:16:09,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-04T19:16:09,864 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T19:16:09,865 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T19:16:09,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-04T19:16:10,016 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-04T19:16:10,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:16:10,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. as already flushing 2024-12-04T19:16:10,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:16:10,017 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:10,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:10,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:10,075 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:10,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52494 deadline: 1733339830072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:10,075 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:10,076 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:10,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52496 deadline: 1733339830072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:10,076 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:10,076 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:10,076 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:10,076 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:10,076 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:10,084 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:10,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52588 deadline: 1733339830081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:10,084 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:10,084 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:10,084 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:10,108 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:10,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52544 deadline: 1733339830102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:10,109 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:10,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52546 deadline: 1733339830102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:10,109 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:10,109 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:10,109 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:10,109 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:10,109 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:10,110 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:10,169 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-04T19:16:10,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:16:10,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. as already flushing 2024-12-04T19:16:10,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:16:10,170 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:10,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:10,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:10,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-04T19:16:10,207 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/C/4d98aac7ba314f2d9d4e397ee7d821c8 2024-12-04T19:16:10,210 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/A/165cafd9b2c649ad9116015865fc4164 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/165cafd9b2c649ad9116015865fc4164 2024-12-04T19:16:10,213 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/165cafd9b2c649ad9116015865fc4164, entries=200, sequenceid=297, filesize=14.4 K 2024-12-04T19:16:10,213 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/B/5cd3638558d94d949c9f0420362bd353 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/5cd3638558d94d949c9f0420362bd353 2024-12-04T19:16:10,216 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/5cd3638558d94d949c9f0420362bd353, entries=150, sequenceid=297, filesize=12.0 K 2024-12-04T19:16:10,216 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/C/4d98aac7ba314f2d9d4e397ee7d821c8 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/4d98aac7ba314f2d9d4e397ee7d821c8 2024-12-04T19:16:10,219 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/4d98aac7ba314f2d9d4e397ee7d821c8, entries=150, sequenceid=297, filesize=12.0 K 2024-12-04T19:16:10,220 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 48c104a4c7a51d8b62d084d2a093043a in 855ms, sequenceid=297, compaction requested=true 2024-12-04T19:16:10,220 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 48c104a4c7a51d8b62d084d2a093043a: 2024-12-04T19:16:10,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 48c104a4c7a51d8b62d084d2a093043a:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T19:16:10,220 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T19:16:10,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:16:10,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 48c104a4c7a51d8b62d084d2a093043a:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T19:16:10,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:16:10,220 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T19:16:10,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 48c104a4c7a51d8b62d084d2a093043a:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T19:16:10,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T19:16:10,220 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39841 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T19:16:10,220 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37401 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T19:16:10,221 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1541): 48c104a4c7a51d8b62d084d2a093043a/A is initiating minor compaction (all files) 2024-12-04T19:16:10,221 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1541): 48c104a4c7a51d8b62d084d2a093043a/B is initiating minor compaction (all files) 2024-12-04T19:16:10,221 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 48c104a4c7a51d8b62d084d2a093043a/A in TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:16:10,221 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 48c104a4c7a51d8b62d084d2a093043a/B in TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:16:10,221 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/9ebae3da0cdd4c85bf191c43c98ac656, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/8e390ba636864a1f946b620f3b447715, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/5cd3638558d94d949c9f0420362bd353] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp, totalSize=36.5 K 2024-12-04T19:16:10,221 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/ab9dd125c77b47539ff2afa30cbeedda, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/282a125926eb423e82beabaf19ba1b47, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/165cafd9b2c649ad9116015865fc4164] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp, totalSize=38.9 K 2024-12-04T19:16:10,221 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting ab9dd125c77b47539ff2afa30cbeedda, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1733339765649 2024-12-04T19:16:10,221 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ebae3da0cdd4c85bf191c43c98ac656, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1733339765649 2024-12-04T19:16:10,221 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e390ba636864a1f946b620f3b447715, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1733339767862 2024-12-04T19:16:10,221 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 282a125926eb423e82beabaf19ba1b47, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1733339767862 2024-12-04T19:16:10,221 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 165cafd9b2c649ad9116015865fc4164, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1733339769037 2024-12-04T19:16:10,221 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 5cd3638558d94d949c9f0420362bd353, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1733339769038 2024-12-04T19:16:10,227 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 48c104a4c7a51d8b62d084d2a093043a#A#compaction#437 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:16:10,227 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 48c104a4c7a51d8b62d084d2a093043a#B#compaction#438 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:16:10,227 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/A/27ce265228494423acae33c8dfb2ab6b is 50, key is test_row_0/A:col10/1733339769038/Put/seqid=0 2024-12-04T19:16:10,227 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/B/a0dbab56b1794da38ff032437238b981 is 50, key is test_row_0/B:col10/1733339769038/Put/seqid=0 2024-12-04T19:16:10,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742338_1514 (size=13051) 2024-12-04T19:16:10,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742339_1515 (size=13051) 2024-12-04T19:16:10,243 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/A/27ce265228494423acae33c8dfb2ab6b as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/27ce265228494423acae33c8dfb2ab6b 2024-12-04T19:16:10,249 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 48c104a4c7a51d8b62d084d2a093043a/A of 48c104a4c7a51d8b62d084d2a093043a into 27ce265228494423acae33c8dfb2ab6b(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:16:10,249 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 48c104a4c7a51d8b62d084d2a093043a: 2024-12-04T19:16:10,249 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., storeName=48c104a4c7a51d8b62d084d2a093043a/A, priority=13, startTime=1733339770220; duration=0sec 2024-12-04T19:16:10,249 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T19:16:10,249 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 48c104a4c7a51d8b62d084d2a093043a:A 2024-12-04T19:16:10,249 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T19:16:10,256 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37401 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T19:16:10,256 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1541): 48c104a4c7a51d8b62d084d2a093043a/C is initiating minor compaction (all files) 2024-12-04T19:16:10,256 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 48c104a4c7a51d8b62d084d2a093043a/C in TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:16:10,256 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/b716959b1102447685288f4963b327cc, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/20010184066c47aba5acd7458944fb85, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/4d98aac7ba314f2d9d4e397ee7d821c8] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp, totalSize=36.5 K 2024-12-04T19:16:10,257 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting b716959b1102447685288f4963b327cc, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1733339765649 2024-12-04T19:16:10,257 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 20010184066c47aba5acd7458944fb85, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1733339767862 2024-12-04T19:16:10,257 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d98aac7ba314f2d9d4e397ee7d821c8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1733339769038 2024-12-04T19:16:10,270 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 48c104a4c7a51d8b62d084d2a093043a#C#compaction#439 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:16:10,270 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/C/731e75cf8406410bae8db2a58f1601c7 is 50, key is test_row_0/C:col10/1733339769038/Put/seqid=0 2024-12-04T19:16:10,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742340_1516 (size=13051) 2024-12-04T19:16:10,322 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-04T19:16:10,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:16:10,322 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing 48c104a4c7a51d8b62d084d2a093043a 3/3 column families, dataSize=134.18 KB heapSize=366.23 KB 2024-12-04T19:16:10,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 48c104a4c7a51d8b62d084d2a093043a, store=A 2024-12-04T19:16:10,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T19:16:10,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 48c104a4c7a51d8b62d084d2a093043a, store=B 2024-12-04T19:16:10,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T19:16:10,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 48c104a4c7a51d8b62d084d2a093043a, store=C 2024-12-04T19:16:10,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T19:16:10,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/A/777476b2530740b685c31df2c50e4cc0 is 50, key is test_row_0/A:col10/1733339769448/Put/seqid=0 2024-12-04T19:16:10,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742341_1517 (size=12301) 2024-12-04T19:16:10,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-04T19:16:10,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(8855): Flush requested on 48c104a4c7a51d8b62d084d2a093043a 2024-12-04T19:16:10,586 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. as already flushing 2024-12-04T19:16:10,614 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:10,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52496 deadline: 1733339830603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:10,615 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:10,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52588 deadline: 1733339830604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:10,615 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:10,615 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:10,615 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:10,615 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:10,616 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:10,616 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:10,616 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:10,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52494 deadline: 1733339830605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:10,617 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:10,617 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:10,617 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:10,630 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:10,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52544 deadline: 1733339830621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:10,631 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:10,631 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:10,631 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:10,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52546 deadline: 1733339830621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:10,631 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:10,631 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:10,631 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:10,631 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:10,643 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/B/a0dbab56b1794da38ff032437238b981 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/a0dbab56b1794da38ff032437238b981 2024-12-04T19:16:10,646 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 48c104a4c7a51d8b62d084d2a093043a/B of 48c104a4c7a51d8b62d084d2a093043a into a0dbab56b1794da38ff032437238b981(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:16:10,646 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 48c104a4c7a51d8b62d084d2a093043a: 2024-12-04T19:16:10,646 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., storeName=48c104a4c7a51d8b62d084d2a093043a/B, priority=13, startTime=1733339770220; duration=0sec 2024-12-04T19:16:10,646 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:16:10,646 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 48c104a4c7a51d8b62d084d2a093043a:B 2024-12-04T19:16:10,685 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/C/731e75cf8406410bae8db2a58f1601c7 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/731e75cf8406410bae8db2a58f1601c7 2024-12-04T19:16:10,688 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 48c104a4c7a51d8b62d084d2a093043a/C of 48c104a4c7a51d8b62d084d2a093043a into 731e75cf8406410bae8db2a58f1601c7(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:16:10,688 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 48c104a4c7a51d8b62d084d2a093043a: 2024-12-04T19:16:10,688 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., storeName=48c104a4c7a51d8b62d084d2a093043a/C, priority=13, startTime=1733339770220; duration=0sec 2024-12-04T19:16:10,688 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:16:10,688 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 48c104a4c7a51d8b62d084d2a093043a:C 2024-12-04T19:16:10,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:10,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52496 deadline: 1733339830721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:10,728 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:10,728 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:10,728 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:10,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52588 deadline: 1733339830721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:10,728 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:10,728 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:10,728 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:10,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52494 deadline: 1733339830721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:10,728 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:10,728 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:10,729 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/A/777476b2530740b685c31df2c50e4cc0 2024-12-04T19:16:10,729 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:10,729 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:10,729 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:10,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/B/b73cb9348a444e5990f7dd68f3fa74e4 is 50, key is test_row_0/B:col10/1733339769448/Put/seqid=0 2024-12-04T19:16:10,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742342_1518 (size=12301) 2024-12-04T19:16:10,738 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/B/b73cb9348a444e5990f7dd68f3fa74e4 2024-12-04T19:16:10,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/C/ae4e5d4ad7ff41118c44e0197ab725d0 is 50, key is test_row_0/C:col10/1733339769448/Put/seqid=0 2024-12-04T19:16:10,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742343_1519 (size=12301) 2024-12-04T19:16:10,937 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:10,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52496 deadline: 1733339830931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:10,937 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:10,937 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:10,937 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:10,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52588 deadline: 1733339830931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:10,937 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:10,937 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:10,938 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:10,937 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:10,938 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:10,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52494 deadline: 1733339830931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:10,938 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:10,938 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:10,938 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:11,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-04T19:16:11,147 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/C/ae4e5d4ad7ff41118c44e0197ab725d0 2024-12-04T19:16:11,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/A/777476b2530740b685c31df2c50e4cc0 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/777476b2530740b685c31df2c50e4cc0 2024-12-04T19:16:11,155 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/777476b2530740b685c31df2c50e4cc0, entries=150, sequenceid=332, filesize=12.0 K 2024-12-04T19:16:11,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/B/b73cb9348a444e5990f7dd68f3fa74e4 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/b73cb9348a444e5990f7dd68f3fa74e4 2024-12-04T19:16:11,159 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/b73cb9348a444e5990f7dd68f3fa74e4, entries=150, sequenceid=332, filesize=12.0 K 2024-12-04T19:16:11,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/C/ae4e5d4ad7ff41118c44e0197ab725d0 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/ae4e5d4ad7ff41118c44e0197ab725d0 2024-12-04T19:16:11,164 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/ae4e5d4ad7ff41118c44e0197ab725d0, entries=150, sequenceid=332, filesize=12.0 K 2024-12-04T19:16:11,165 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~366.19 KB/374976, currentSize=53.67 KB/54960 for 48c104a4c7a51d8b62d084d2a093043a in 843ms, sequenceid=332, compaction requested=false 2024-12-04T19:16:11,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for 48c104a4c7a51d8b62d084d2a093043a: 2024-12-04T19:16:11,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:16:11,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-12-04T19:16:11,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-12-04T19:16:11,172 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-12-04T19:16:11,172 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3030 sec 2024-12-04T19:16:11,174 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 1.3110 sec 2024-12-04T19:16:11,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(8855): Flush requested on 48c104a4c7a51d8b62d084d2a093043a 2024-12-04T19:16:11,244 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 48c104a4c7a51d8b62d084d2a093043a 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-04T19:16:11,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 48c104a4c7a51d8b62d084d2a093043a, store=A 2024-12-04T19:16:11,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:16:11,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 48c104a4c7a51d8b62d084d2a093043a, store=B 2024-12-04T19:16:11,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:16:11,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 48c104a4c7a51d8b62d084d2a093043a, store=C 2024-12-04T19:16:11,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:16:11,248 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/A/dbfca6ebceab4c89b21f88d25187f250 is 50, key is test_row_0/A:col10/1733339770600/Put/seqid=0 2024-12-04T19:16:11,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742344_1520 (size=14741) 2024-12-04T19:16:11,295 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:11,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52496 deadline: 1733339831292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:11,296 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:11,296 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:11,296 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:11,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:11,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52588 deadline: 1733339831295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:11,303 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:11,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52494 deadline: 1733339831295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:11,304 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:11,304 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:11,304 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:11,304 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:11,304 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:11,304 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:11,406 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:11,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52496 deadline: 1733339831401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:11,407 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:11,407 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:11,407 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:11,419 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:11,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52494 deadline: 1733339831412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:11,420 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:11,420 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:11,420 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:11,420 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:11,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52588 deadline: 1733339831412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:11,421 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:11,421 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:11,421 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:11,617 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:11,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52496 deadline: 1733339831611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:11,617 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:11,617 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:11,617 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:11,624 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:11,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52494 deadline: 1733339831622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:11,624 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:11,624 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:11,624 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:11,638 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:11,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52588 deadline: 1733339831632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:11,638 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:11,638 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:11,638 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:11,647 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:11,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52544 deadline: 1733339831642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:11,647 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:11,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52546 deadline: 1733339831642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:11,647 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 9ba2734cd39f,33969,1733339629791 for 'test_row_0' in 48c104a4c7a51d8b62d084d2a093043a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2198 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-04T19:16:11,647 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 9ba2734cd39f,33969,1733339629791 for 'test_row_1' in 48c104a4c7a51d8b62d084d2a093043a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2198 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-04T19:16:11,648 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:11,648 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:11,648 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:11,648 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:11,648 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:11,648 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:11,658 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/A/dbfca6ebceab4c89b21f88d25187f250 2024-12-04T19:16:11,663 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/B/2545b4e93ece49b1a2374a75a8787ce3 is 50, key is test_row_0/B:col10/1733339770600/Put/seqid=0 2024-12-04T19:16:11,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742345_1521 (size=12301) 2024-12-04T19:16:11,926 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:11,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52496 deadline: 1733339831921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:11,926 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:11,927 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:11,927 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:11,935 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:11,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52494 deadline: 1733339831931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:11,935 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:11,935 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:11,935 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:11,957 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:11,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52588 deadline: 1733339831951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:11,961 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:11,961 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:11,961 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:12,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-04T19:16:12,011 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T19:16:12,012 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T19:16:12,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-12-04T19:16:12,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-04T19:16:12,013 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T19:16:12,014 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T19:16:12,014 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T19:16:12,066 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/B/2545b4e93ece49b1a2374a75a8787ce3 2024-12-04T19:16:12,075 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/C/abd0b586f3804270b14b1b3f03370180 is 50, key is test_row_0/C:col10/1733339770600/Put/seqid=0 2024-12-04T19:16:12,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742346_1522 (size=12301) 2024-12-04T19:16:12,079 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/C/abd0b586f3804270b14b1b3f03370180 2024-12-04T19:16:12,081 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/A/dbfca6ebceab4c89b21f88d25187f250 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/dbfca6ebceab4c89b21f88d25187f250 2024-12-04T19:16:12,084 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/dbfca6ebceab4c89b21f88d25187f250, entries=200, sequenceid=346, filesize=14.4 K 2024-12-04T19:16:12,084 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/B/2545b4e93ece49b1a2374a75a8787ce3 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/2545b4e93ece49b1a2374a75a8787ce3 2024-12-04T19:16:12,087 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/2545b4e93ece49b1a2374a75a8787ce3, entries=150, sequenceid=346, filesize=12.0 K 2024-12-04T19:16:12,088 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/C/abd0b586f3804270b14b1b3f03370180 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/abd0b586f3804270b14b1b3f03370180 2024-12-04T19:16:12,091 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/abd0b586f3804270b14b1b3f03370180, entries=150, sequenceid=346, filesize=12.0 K 2024-12-04T19:16:12,092 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 48c104a4c7a51d8b62d084d2a093043a in 848ms, sequenceid=346, compaction requested=true 2024-12-04T19:16:12,092 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 48c104a4c7a51d8b62d084d2a093043a: 2024-12-04T19:16:12,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 48c104a4c7a51d8b62d084d2a093043a:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T19:16:12,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:16:12,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 48c104a4c7a51d8b62d084d2a093043a:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T19:16:12,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:16:12,092 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T19:16:12,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 48c104a4c7a51d8b62d084d2a093043a:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T19:16:12,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T19:16:12,092 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T19:16:12,093 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T19:16:12,093 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40093 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T19:16:12,093 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1541): 48c104a4c7a51d8b62d084d2a093043a/B is initiating minor compaction (all files) 2024-12-04T19:16:12,093 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1541): 48c104a4c7a51d8b62d084d2a093043a/A is initiating minor compaction (all files) 2024-12-04T19:16:12,093 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 48c104a4c7a51d8b62d084d2a093043a/B in TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:16:12,093 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 48c104a4c7a51d8b62d084d2a093043a/A in TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:16:12,093 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/a0dbab56b1794da38ff032437238b981, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/b73cb9348a444e5990f7dd68f3fa74e4, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/2545b4e93ece49b1a2374a75a8787ce3] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp, totalSize=36.8 K 2024-12-04T19:16:12,093 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/27ce265228494423acae33c8dfb2ab6b, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/777476b2530740b685c31df2c50e4cc0, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/dbfca6ebceab4c89b21f88d25187f250] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp, totalSize=39.2 K 2024-12-04T19:16:12,093 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting a0dbab56b1794da38ff032437238b981, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1733339769038 2024-12-04T19:16:12,093 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 27ce265228494423acae33c8dfb2ab6b, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1733339769038 2024-12-04T19:16:12,095 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 777476b2530740b685c31df2c50e4cc0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1733339769437 2024-12-04T19:16:12,095 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting b73cb9348a444e5990f7dd68f3fa74e4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1733339769437 2024-12-04T19:16:12,095 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 2545b4e93ece49b1a2374a75a8787ce3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1733339770600 2024-12-04T19:16:12,095 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting dbfca6ebceab4c89b21f88d25187f250, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1733339770600 2024-12-04T19:16:12,101 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 48c104a4c7a51d8b62d084d2a093043a#B#compaction#446 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:16:12,101 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 48c104a4c7a51d8b62d084d2a093043a#A#compaction#447 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:16:12,102 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/A/21c2540f5e6d4e35b85c7089a75df00c is 50, key is test_row_0/A:col10/1733339770600/Put/seqid=0 2024-12-04T19:16:12,102 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/B/affb887500934a77a994e827e9868ad7 is 50, key is test_row_0/B:col10/1733339770600/Put/seqid=0 2024-12-04T19:16:12,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742348_1524 (size=13153) 2024-12-04T19:16:12,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742347_1523 (size=13153) 2024-12-04T19:16:12,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-04T19:16:12,126 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/A/21c2540f5e6d4e35b85c7089a75df00c as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/21c2540f5e6d4e35b85c7089a75df00c 2024-12-04T19:16:12,134 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 48c104a4c7a51d8b62d084d2a093043a/A of 48c104a4c7a51d8b62d084d2a093043a into 21c2540f5e6d4e35b85c7089a75df00c(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:16:12,134 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 48c104a4c7a51d8b62d084d2a093043a: 2024-12-04T19:16:12,134 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., storeName=48c104a4c7a51d8b62d084d2a093043a/A, priority=13, startTime=1733339772092; duration=0sec 2024-12-04T19:16:12,134 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T19:16:12,134 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 48c104a4c7a51d8b62d084d2a093043a:A 2024-12-04T19:16:12,134 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T19:16:12,135 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T19:16:12,136 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1541): 48c104a4c7a51d8b62d084d2a093043a/C is initiating minor compaction (all files) 2024-12-04T19:16:12,136 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 48c104a4c7a51d8b62d084d2a093043a/C in TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:16:12,136 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/731e75cf8406410bae8db2a58f1601c7, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/ae4e5d4ad7ff41118c44e0197ab725d0, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/abd0b586f3804270b14b1b3f03370180] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp, totalSize=36.8 K 2024-12-04T19:16:12,136 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 731e75cf8406410bae8db2a58f1601c7, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1733339769038 2024-12-04T19:16:12,137 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae4e5d4ad7ff41118c44e0197ab725d0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1733339769437 2024-12-04T19:16:12,137 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting abd0b586f3804270b14b1b3f03370180, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1733339770600 2024-12-04T19:16:12,146 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 48c104a4c7a51d8b62d084d2a093043a#C#compaction#448 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:16:12,147 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/C/6fdff6a5b77d4ee5acd0bbe6d6e841fe is 50, key is test_row_0/C:col10/1733339770600/Put/seqid=0 2024-12-04T19:16:12,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742349_1525 (size=13153) 2024-12-04T19:16:12,166 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-12-04T19:16:12,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:16:12,166 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing 48c104a4c7a51d8b62d084d2a093043a 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-04T19:16:12,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 48c104a4c7a51d8b62d084d2a093043a, store=A 2024-12-04T19:16:12,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:16:12,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 48c104a4c7a51d8b62d084d2a093043a, store=B 2024-12-04T19:16:12,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:16:12,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 48c104a4c7a51d8b62d084d2a093043a, store=C 2024-12-04T19:16:12,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:16:12,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/A/65e880b2956945d69162cf031b8c4105 is 50, key is test_row_0/A:col10/1733339771292/Put/seqid=0 2024-12-04T19:16:12,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742350_1526 (size=12301) 2024-12-04T19:16:12,176 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/A/65e880b2956945d69162cf031b8c4105 2024-12-04T19:16:12,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/B/a352dde772fa45d2bd45f911ff5c6926 is 50, key is test_row_0/B:col10/1733339771292/Put/seqid=0 2024-12-04T19:16:12,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742351_1527 (size=12301) 2024-12-04T19:16:12,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-04T19:16:12,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(8855): Flush requested on 48c104a4c7a51d8b62d084d2a093043a 2024-12-04T19:16:12,433 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. as already flushing 2024-12-04T19:16:12,461 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:12,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52494 deadline: 1733339832456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:12,461 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:12,461 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:12,461 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:12,462 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:12,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52496 deadline: 1733339832457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:12,463 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:12,463 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:12,463 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:12,477 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:12,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52588 deadline: 1733339832471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:12,477 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:12,477 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:12,477 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:12,513 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/B/affb887500934a77a994e827e9868ad7 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/affb887500934a77a994e827e9868ad7 2024-12-04T19:16:12,516 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 48c104a4c7a51d8b62d084d2a093043a/B of 48c104a4c7a51d8b62d084d2a093043a into affb887500934a77a994e827e9868ad7(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:16:12,516 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 48c104a4c7a51d8b62d084d2a093043a: 2024-12-04T19:16:12,516 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., storeName=48c104a4c7a51d8b62d084d2a093043a/B, priority=13, startTime=1733339772092; duration=0sec 2024-12-04T19:16:12,516 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:16:12,516 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 48c104a4c7a51d8b62d084d2a093043a:B 2024-12-04T19:16:12,562 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/C/6fdff6a5b77d4ee5acd0bbe6d6e841fe as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/6fdff6a5b77d4ee5acd0bbe6d6e841fe 2024-12-04T19:16:12,565 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 48c104a4c7a51d8b62d084d2a093043a/C of 48c104a4c7a51d8b62d084d2a093043a into 6fdff6a5b77d4ee5acd0bbe6d6e841fe(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:16:12,566 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 48c104a4c7a51d8b62d084d2a093043a: 2024-12-04T19:16:12,566 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., storeName=48c104a4c7a51d8b62d084d2a093043a/C, priority=13, startTime=1733339772092; duration=0sec 2024-12-04T19:16:12,566 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:16:12,566 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 48c104a4c7a51d8b62d084d2a093043a:C 2024-12-04T19:16:12,575 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:12,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52494 deadline: 1733339832571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:12,576 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:12,576 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:12,576 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:12,576 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:12,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52496 deadline: 1733339832571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:12,576 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:12,576 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:12,576 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:12,586 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/B/a352dde772fa45d2bd45f911ff5c6926 2024-12-04T19:16:12,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/C/0b1c5b2b6700446b9e5eadbe3598b9e8 is 50, key is test_row_0/C:col10/1733339771292/Put/seqid=0 2024-12-04T19:16:12,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742352_1528 (size=12301) 2024-12-04T19:16:12,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-04T19:16:12,786 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:12,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52494 deadline: 1733339832782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:12,786 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:12,786 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:12,786 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:12,786 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:12,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52496 deadline: 1733339832782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:12,787 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 , the old value is region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:12,787 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:12,787 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., hostname=9ba2734cd39f,33969,1733339629791, seqNum=2 because the exception is null or not the one we care about 2024-12-04T19:16:12,994 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/C/0b1c5b2b6700446b9e5eadbe3598b9e8 2024-12-04T19:16:12,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/A/65e880b2956945d69162cf031b8c4105 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/65e880b2956945d69162cf031b8c4105 2024-12-04T19:16:12,999 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/65e880b2956945d69162cf031b8c4105, entries=150, sequenceid=367, filesize=12.0 K 2024-12-04T19:16:13,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/B/a352dde772fa45d2bd45f911ff5c6926 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/a352dde772fa45d2bd45f911ff5c6926 2024-12-04T19:16:13,003 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/a352dde772fa45d2bd45f911ff5c6926, entries=150, sequenceid=367, filesize=12.0 K 2024-12-04T19:16:13,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/C/0b1c5b2b6700446b9e5eadbe3598b9e8 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/0b1c5b2b6700446b9e5eadbe3598b9e8 2024-12-04T19:16:13,007 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/0b1c5b2b6700446b9e5eadbe3598b9e8, entries=150, sequenceid=367, filesize=12.0 K 2024-12-04T19:16:13,008 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 48c104a4c7a51d8b62d084d2a093043a in 842ms, sequenceid=367, compaction requested=false 2024-12-04T19:16:13,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for 48c104a4c7a51d8b62d084d2a093043a: 2024-12-04T19:16:13,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:16:13,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-12-04T19:16:13,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-12-04T19:16:13,011 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-12-04T19:16:13,011 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 995 msec 2024-12-04T19:16:13,012 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 999 msec 2024-12-04T19:16:13,058 INFO [Thread-2016 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2016. 2024-12-04T19:16:13,058 DEBUG [Thread-2016 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T19:16:13,058 DEBUG [Thread-2016 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:16:13,058 DEBUG [Thread-2016 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:16:13,058 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T19:16:13,059 INFO [Thread-2020 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2020. 2024-12-04T19:16:13,059 DEBUG [Thread-2020 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T19:16:13,059 DEBUG [Thread-2020 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:16:13,059 DEBUG [Thread-2020 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:16:13,059 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T19:16:13,061 INFO [Thread-2018 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2018. 2024-12-04T19:16:13,061 DEBUG [Thread-2018 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T19:16:13,061 DEBUG [Thread-2018 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:16:13,061 DEBUG [Thread-2018 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:16:13,061 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T19:16:13,062 INFO [Thread-2022 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2022. 2024-12-04T19:16:13,062 DEBUG [Thread-2022 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T19:16:13,062 DEBUG [Thread-2022 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:16:13,062 DEBUG [Thread-2022 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:16:13,062 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T19:16:13,063 INFO [Thread-2014 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2014. 2024-12-04T19:16:13,063 DEBUG [Thread-2014 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T19:16:13,063 DEBUG [Thread-2014 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:16:13,063 DEBUG [Thread-2014 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:16:13,063 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T19:16:13,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(8855): Flush requested on 48c104a4c7a51d8b62d084d2a093043a 2024-12-04T19:16:13,094 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 48c104a4c7a51d8b62d084d2a093043a 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-04T19:16:13,095 INFO [Thread-2009 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2009. 2024-12-04T19:16:13,095 INFO [Thread-2005 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2005. 2024-12-04T19:16:13,095 DEBUG [Thread-2009 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T19:16:13,095 DEBUG [Thread-2005 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T19:16:13,095 DEBUG [Thread-2009 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:16:13,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 48c104a4c7a51d8b62d084d2a093043a, store=A 2024-12-04T19:16:13,095 DEBUG [Thread-2005 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:16:13,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:16:13,095 DEBUG [Thread-2009 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:16:13,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 48c104a4c7a51d8b62d084d2a093043a, store=B 2024-12-04T19:16:13,095 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T19:16:13,095 DEBUG [Thread-2005 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:16:13,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:16:13,095 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T19:16:13,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 48c104a4c7a51d8b62d084d2a093043a, store=C 2024-12-04T19:16:13,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:16:13,100 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/A/d1527bc8a7544041adbd8034297979ff is 50, key is test_row_0/A:col10/1733339772456/Put/seqid=0 2024-12-04T19:16:13,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742353_1529 (size=12301) 2024-12-04T19:16:13,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-04T19:16:13,152 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T19:16:13,485 INFO [Thread-2007 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2007. 2024-12-04T19:16:13,485 DEBUG [Thread-2007 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T19:16:13,485 DEBUG [Thread-2007 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:16:13,486 DEBUG [Thread-2007 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:16:13,486 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T19:16:13,506 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=383 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/A/d1527bc8a7544041adbd8034297979ff 2024-12-04T19:16:13,519 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/B/a91923fe33454a9093cacd03b2d71f59 is 50, key is test_row_0/B:col10/1733339772456/Put/seqid=0 2024-12-04T19:16:13,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742354_1530 (size=12301) 2024-12-04T19:16:13,654 INFO [Thread-2003 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2003. 2024-12-04T19:16:13,654 DEBUG [Thread-2003 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T19:16:13,654 DEBUG [Thread-2003 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:16:13,654 DEBUG [Thread-2003 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:16:13,654 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T19:16:13,663 INFO [Thread-2011 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2011. 2024-12-04T19:16:13,663 DEBUG [Thread-2011 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T19:16:13,663 DEBUG [Thread-2011 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:16:13,663 DEBUG [Thread-2011 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:16:13,663 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T19:16:13,664 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-04T19:16:13,664 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 50 2024-12-04T19:16:13,664 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 60 2024-12-04T19:16:13,664 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 69 2024-12-04T19:16:13,664 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 63 2024-12-04T19:16:13,664 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 59 2024-12-04T19:16:13,664 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-04T19:16:13,664 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-04T19:16:13,664 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1651 2024-12-04T19:16:13,664 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4949 rows 2024-12-04T19:16:13,664 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1649 2024-12-04T19:16:13,664 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4944 rows 2024-12-04T19:16:13,664 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1650 2024-12-04T19:16:13,664 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4946 rows 2024-12-04T19:16:13,664 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1635 2024-12-04T19:16:13,664 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4899 rows 2024-12-04T19:16:13,664 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1653 2024-12-04T19:16:13,664 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4959 rows 2024-12-04T19:16:13,664 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-04T19:16:13,664 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T19:16:13,664 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:16:13,664 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:16:13,665 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T19:16:13,665 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-04T19:16:13,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-04T19:16:13,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-04T19:16:13,670 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733339773670"}]},"ts":"1733339773670"} 2024-12-04T19:16:13,672 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-04T19:16:13,672 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-04T19:16:13,673 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-04T19:16:13,675 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=48c104a4c7a51d8b62d084d2a093043a, UNASSIGN}] 2024-12-04T19:16:13,676 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=48c104a4c7a51d8b62d084d2a093043a, UNASSIGN 2024-12-04T19:16:13,677 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=48c104a4c7a51d8b62d084d2a093043a, regionState=CLOSING, regionLocation=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:13,679 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=48c104a4c7a51d8b62d084d2a093043a, UNASSIGN because future has completed 2024-12-04T19:16:13,679 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-04T19:16:13,679 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; CloseRegionProcedure 48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791}] 2024-12-04T19:16:13,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-04T19:16:13,832 INFO [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] handler.UnassignRegionHandler(122): Close 48c104a4c7a51d8b62d084d2a093043a 2024-12-04T19:16:13,832 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-04T19:16:13,832 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1722): Closing 48c104a4c7a51d8b62d084d2a093043a, disabling compactions & flushes 2024-12-04T19:16:13,832 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:16:13,925 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=383 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/B/a91923fe33454a9093cacd03b2d71f59 2024-12-04T19:16:13,930 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/C/48567464762b4ff28caaccce0629579f is 50, key is test_row_0/C:col10/1733339772456/Put/seqid=0 2024-12-04T19:16:13,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742355_1531 (size=12301) 2024-12-04T19:16:13,934 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=383 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/C/48567464762b4ff28caaccce0629579f 2024-12-04T19:16:13,937 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/A/d1527bc8a7544041adbd8034297979ff as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/d1527bc8a7544041adbd8034297979ff 2024-12-04T19:16:13,939 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/d1527bc8a7544041adbd8034297979ff, entries=150, sequenceid=383, filesize=12.0 K 2024-12-04T19:16:13,940 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/B/a91923fe33454a9093cacd03b2d71f59 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/a91923fe33454a9093cacd03b2d71f59 2024-12-04T19:16:13,942 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/a91923fe33454a9093cacd03b2d71f59, entries=150, sequenceid=383, filesize=12.0 K 2024-12-04T19:16:13,942 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/C/48567464762b4ff28caaccce0629579f as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/48567464762b4ff28caaccce0629579f 2024-12-04T19:16:13,945 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/48567464762b4ff28caaccce0629579f, entries=150, sequenceid=383, filesize=12.0 K 2024-12-04T19:16:13,946 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=20.13 KB/20610 for 48c104a4c7a51d8b62d084d2a093043a in 852ms, sequenceid=383, compaction requested=true 2024-12-04T19:16:13,946 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 48c104a4c7a51d8b62d084d2a093043a: 2024-12-04T19:16:13,946 INFO [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:16:13,946 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:16:13,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 48c104a4c7a51d8b62d084d2a093043a:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T19:16:13,946 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. after waiting 0 ms 2024-12-04T19:16:13,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:16:13,946 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:16:13,946 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. because compaction request was cancelled 2024-12-04T19:16:13,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 48c104a4c7a51d8b62d084d2a093043a:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T19:16:13,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:16:13,946 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 48c104a4c7a51d8b62d084d2a093043a:A 2024-12-04T19:16:13,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 48c104a4c7a51d8b62d084d2a093043a:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T19:16:13,946 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. because compaction request was cancelled 2024-12-04T19:16:13,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:16:13,946 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. because compaction request was cancelled 2024-12-04T19:16:13,946 INFO [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(2902): Flushing 48c104a4c7a51d8b62d084d2a093043a 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-12-04T19:16:13,946 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 48c104a4c7a51d8b62d084d2a093043a:B 2024-12-04T19:16:13,946 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 48c104a4c7a51d8b62d084d2a093043a:C 2024-12-04T19:16:13,946 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 48c104a4c7a51d8b62d084d2a093043a, store=A 2024-12-04T19:16:13,946 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:16:13,946 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 48c104a4c7a51d8b62d084d2a093043a, store=B 2024-12-04T19:16:13,946 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:16:13,946 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 48c104a4c7a51d8b62d084d2a093043a, store=C 2024-12-04T19:16:13,946 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:16:13,949 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/A/f054f83c06834373b25421ecd37ca0fc is 50, key is test_row_0/A:col10/1733339773652/Put/seqid=0 2024-12-04T19:16:13,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742356_1532 (size=9857) 2024-12-04T19:16:13,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-04T19:16:14,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-04T19:16:14,352 INFO [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=389 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/A/f054f83c06834373b25421ecd37ca0fc 2024-12-04T19:16:14,357 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/B/36b3643e067f471289f0966ab37d8fca is 50, key is test_row_0/B:col10/1733339773652/Put/seqid=0 2024-12-04T19:16:14,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742357_1533 (size=9857) 2024-12-04T19:16:14,761 INFO [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=389 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/B/36b3643e067f471289f0966ab37d8fca 2024-12-04T19:16:14,769 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/C/160324cf34dd44d7842ce348da97be0e is 50, key is test_row_0/C:col10/1733339773652/Put/seqid=0 2024-12-04T19:16:14,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742358_1534 (size=9857) 2024-12-04T19:16:14,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-04T19:16:15,177 INFO [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=389 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/C/160324cf34dd44d7842ce348da97be0e 2024-12-04T19:16:15,182 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/A/f054f83c06834373b25421ecd37ca0fc as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/f054f83c06834373b25421ecd37ca0fc 2024-12-04T19:16:15,185 INFO [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/f054f83c06834373b25421ecd37ca0fc, entries=100, sequenceid=389, filesize=9.6 K 2024-12-04T19:16:15,186 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/B/36b3643e067f471289f0966ab37d8fca as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/36b3643e067f471289f0966ab37d8fca 2024-12-04T19:16:15,188 INFO [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/36b3643e067f471289f0966ab37d8fca, entries=100, sequenceid=389, filesize=9.6 K 2024-12-04T19:16:15,188 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/.tmp/C/160324cf34dd44d7842ce348da97be0e as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/160324cf34dd44d7842ce348da97be0e 2024-12-04T19:16:15,191 INFO [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/160324cf34dd44d7842ce348da97be0e, entries=100, sequenceid=389, filesize=9.6 K 2024-12-04T19:16:15,191 INFO [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 48c104a4c7a51d8b62d084d2a093043a in 1245ms, sequenceid=389, compaction requested=true 2024-12-04T19:16:15,192 DEBUG [StoreCloser-TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/9febb8cc21dd47f695ba56f16409c0aa, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/8909666709d74547b527b560d0e9903e, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/c43a493531cb4af5bb9867891dc13df2, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/cfc9b30527eb4443bee32764b34fe9b6, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/eb7c0d547f724ed19113db60a6d47ee3, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/57da3c4c57bf4373826531b6d7f7b684, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/33dbdd994f0640e08417ec2494dcfc82, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/bd9707808aaf42808faed48fd509f8d1, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/9b435a62e9b749f88182df453e4c895e, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/df90f1337ba54599bd1c10c45803e2f7, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/db79c289b4854195b97f58a565038847, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/41555f355f40422b8724c072fbdd035a, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/fc684417244845f7ae4bb5ed1a55d62f, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/e3afc2777545437aa91d15a793aac0dc, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/ab05112c0eb1477788bb02bee9ca6b0f, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/f4ebbc78104f4115a1cc65303850076c, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/e792f178edb64030abbe344ec3d6f3e0, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/104dd64c7ec24a33bb3986099affa060, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/ab9dd125c77b47539ff2afa30cbeedda, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/60dc8d2693a84c97b9e670b0779beecc, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/282a125926eb423e82beabaf19ba1b47, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/165cafd9b2c649ad9116015865fc4164, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/27ce265228494423acae33c8dfb2ab6b, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/777476b2530740b685c31df2c50e4cc0, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/dbfca6ebceab4c89b21f88d25187f250] to archive 2024-12-04T19:16:15,193 DEBUG [StoreCloser-TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-04T19:16:15,194 DEBUG [StoreCloser-TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/9febb8cc21dd47f695ba56f16409c0aa to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/9febb8cc21dd47f695ba56f16409c0aa 2024-12-04T19:16:15,195 DEBUG [StoreCloser-TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/8909666709d74547b527b560d0e9903e to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/8909666709d74547b527b560d0e9903e 2024-12-04T19:16:15,196 DEBUG [StoreCloser-TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/c43a493531cb4af5bb9867891dc13df2 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/c43a493531cb4af5bb9867891dc13df2 2024-12-04T19:16:15,196 DEBUG [StoreCloser-TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/cfc9b30527eb4443bee32764b34fe9b6 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/cfc9b30527eb4443bee32764b34fe9b6 2024-12-04T19:16:15,197 DEBUG [StoreCloser-TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/eb7c0d547f724ed19113db60a6d47ee3 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/eb7c0d547f724ed19113db60a6d47ee3 2024-12-04T19:16:15,198 DEBUG [StoreCloser-TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/57da3c4c57bf4373826531b6d7f7b684 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/57da3c4c57bf4373826531b6d7f7b684 2024-12-04T19:16:15,198 DEBUG [StoreCloser-TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/33dbdd994f0640e08417ec2494dcfc82 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/33dbdd994f0640e08417ec2494dcfc82 2024-12-04T19:16:15,199 DEBUG [StoreCloser-TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/bd9707808aaf42808faed48fd509f8d1 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/bd9707808aaf42808faed48fd509f8d1 2024-12-04T19:16:15,200 DEBUG [StoreCloser-TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/9b435a62e9b749f88182df453e4c895e to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/9b435a62e9b749f88182df453e4c895e 2024-12-04T19:16:15,200 DEBUG [StoreCloser-TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/df90f1337ba54599bd1c10c45803e2f7 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/df90f1337ba54599bd1c10c45803e2f7 2024-12-04T19:16:15,201 DEBUG [StoreCloser-TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/db79c289b4854195b97f58a565038847 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/db79c289b4854195b97f58a565038847 2024-12-04T19:16:15,202 DEBUG [StoreCloser-TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/41555f355f40422b8724c072fbdd035a to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/41555f355f40422b8724c072fbdd035a 2024-12-04T19:16:15,202 DEBUG [StoreCloser-TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/fc684417244845f7ae4bb5ed1a55d62f to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/fc684417244845f7ae4bb5ed1a55d62f 2024-12-04T19:16:15,203 DEBUG [StoreCloser-TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/e3afc2777545437aa91d15a793aac0dc to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/e3afc2777545437aa91d15a793aac0dc 2024-12-04T19:16:15,203 DEBUG [StoreCloser-TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/ab05112c0eb1477788bb02bee9ca6b0f to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/ab05112c0eb1477788bb02bee9ca6b0f 2024-12-04T19:16:15,204 DEBUG [StoreCloser-TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/f4ebbc78104f4115a1cc65303850076c to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/f4ebbc78104f4115a1cc65303850076c 2024-12-04T19:16:15,205 DEBUG [StoreCloser-TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/e792f178edb64030abbe344ec3d6f3e0 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/e792f178edb64030abbe344ec3d6f3e0 2024-12-04T19:16:15,206 DEBUG [StoreCloser-TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/104dd64c7ec24a33bb3986099affa060 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/104dd64c7ec24a33bb3986099affa060 2024-12-04T19:16:15,206 DEBUG [StoreCloser-TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/ab9dd125c77b47539ff2afa30cbeedda to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/ab9dd125c77b47539ff2afa30cbeedda 2024-12-04T19:16:15,207 DEBUG [StoreCloser-TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/60dc8d2693a84c97b9e670b0779beecc to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/60dc8d2693a84c97b9e670b0779beecc 2024-12-04T19:16:15,208 DEBUG [StoreCloser-TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/282a125926eb423e82beabaf19ba1b47 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/282a125926eb423e82beabaf19ba1b47 2024-12-04T19:16:15,208 DEBUG [StoreCloser-TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/165cafd9b2c649ad9116015865fc4164 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/165cafd9b2c649ad9116015865fc4164 2024-12-04T19:16:15,209 DEBUG [StoreCloser-TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/27ce265228494423acae33c8dfb2ab6b to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/27ce265228494423acae33c8dfb2ab6b 2024-12-04T19:16:15,210 DEBUG [StoreCloser-TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/777476b2530740b685c31df2c50e4cc0 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/777476b2530740b685c31df2c50e4cc0 2024-12-04T19:16:15,210 DEBUG [StoreCloser-TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/dbfca6ebceab4c89b21f88d25187f250 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/dbfca6ebceab4c89b21f88d25187f250 2024-12-04T19:16:15,211 DEBUG [StoreCloser-TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/2e1082f0347e42afad92ca5855a1e4ed, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/883ae9d5e2a84ce1a45c75da78312688, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/8d7f81909fba4b99b9500dd98a2e81c7, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/c0c41ec9f65e44b280436206b7d9e2f2, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/8794873c6c724f65b5f7ade5865081bb, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/f2b361ac91034406943633ea9ba76810, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/8d6d4277649c48aca8e1bc5b7b3ecc0c, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/bc0512c2a1bd4cb787b397d9f2d1002f, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/11a6907ef0c64f44b73d9a4f5f8b9b65, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/dc4a5b7de29446ac9b97c5e7fe135d67, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/80252d0e2ca6455ba0189160b4ed543c, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/08cbe18658a64160ba533074ce25a570, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/fd0eabb2bb13442f9674df7bf6d1894c, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/b4273718129a4229b2bb7235ca322c80, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/19a41947da134437805880ea6a94665f, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/e4ee6cb89cde49039f135729b6a8e4bd, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/d2350030940b4135bce171c03373bb27, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/e5713631593b4afebea6af01da58e0de, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/9ebae3da0cdd4c85bf191c43c98ac656, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/e22741162ffd40ccbc4e1a818baf754c, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/8e390ba636864a1f946b620f3b447715, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/a0dbab56b1794da38ff032437238b981, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/5cd3638558d94d949c9f0420362bd353, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/b73cb9348a444e5990f7dd68f3fa74e4, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/2545b4e93ece49b1a2374a75a8787ce3] to archive 2024-12-04T19:16:15,212 DEBUG [StoreCloser-TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-04T19:16:15,213 DEBUG [StoreCloser-TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/2e1082f0347e42afad92ca5855a1e4ed to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/2e1082f0347e42afad92ca5855a1e4ed 2024-12-04T19:16:15,214 DEBUG [StoreCloser-TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/883ae9d5e2a84ce1a45c75da78312688 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/883ae9d5e2a84ce1a45c75da78312688 2024-12-04T19:16:15,214 DEBUG [StoreCloser-TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/8d7f81909fba4b99b9500dd98a2e81c7 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/8d7f81909fba4b99b9500dd98a2e81c7 2024-12-04T19:16:15,215 DEBUG [StoreCloser-TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/c0c41ec9f65e44b280436206b7d9e2f2 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/c0c41ec9f65e44b280436206b7d9e2f2 2024-12-04T19:16:15,216 DEBUG [StoreCloser-TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/8794873c6c724f65b5f7ade5865081bb to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/8794873c6c724f65b5f7ade5865081bb 2024-12-04T19:16:15,216 DEBUG [StoreCloser-TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/f2b361ac91034406943633ea9ba76810 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/f2b361ac91034406943633ea9ba76810 2024-12-04T19:16:15,217 DEBUG [StoreCloser-TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/8d6d4277649c48aca8e1bc5b7b3ecc0c to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/8d6d4277649c48aca8e1bc5b7b3ecc0c 2024-12-04T19:16:15,218 DEBUG [StoreCloser-TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/bc0512c2a1bd4cb787b397d9f2d1002f to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/bc0512c2a1bd4cb787b397d9f2d1002f 2024-12-04T19:16:15,218 DEBUG [StoreCloser-TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/11a6907ef0c64f44b73d9a4f5f8b9b65 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/11a6907ef0c64f44b73d9a4f5f8b9b65 2024-12-04T19:16:15,219 DEBUG [StoreCloser-TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/dc4a5b7de29446ac9b97c5e7fe135d67 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/dc4a5b7de29446ac9b97c5e7fe135d67 2024-12-04T19:16:15,219 DEBUG [StoreCloser-TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/80252d0e2ca6455ba0189160b4ed543c to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/80252d0e2ca6455ba0189160b4ed543c 2024-12-04T19:16:15,220 DEBUG [StoreCloser-TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/08cbe18658a64160ba533074ce25a570 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/08cbe18658a64160ba533074ce25a570 2024-12-04T19:16:15,221 DEBUG [StoreCloser-TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/fd0eabb2bb13442f9674df7bf6d1894c to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/fd0eabb2bb13442f9674df7bf6d1894c 2024-12-04T19:16:15,221 DEBUG [StoreCloser-TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/b4273718129a4229b2bb7235ca322c80 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/b4273718129a4229b2bb7235ca322c80 2024-12-04T19:16:15,222 DEBUG [StoreCloser-TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/19a41947da134437805880ea6a94665f to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/19a41947da134437805880ea6a94665f 2024-12-04T19:16:15,222 DEBUG [StoreCloser-TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/e4ee6cb89cde49039f135729b6a8e4bd to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/e4ee6cb89cde49039f135729b6a8e4bd 2024-12-04T19:16:15,223 DEBUG [StoreCloser-TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/d2350030940b4135bce171c03373bb27 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/d2350030940b4135bce171c03373bb27 2024-12-04T19:16:15,224 DEBUG [StoreCloser-TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/e5713631593b4afebea6af01da58e0de to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/e5713631593b4afebea6af01da58e0de 2024-12-04T19:16:15,225 DEBUG [StoreCloser-TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/9ebae3da0cdd4c85bf191c43c98ac656 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/9ebae3da0cdd4c85bf191c43c98ac656 2024-12-04T19:16:15,225 DEBUG [StoreCloser-TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/e22741162ffd40ccbc4e1a818baf754c to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/e22741162ffd40ccbc4e1a818baf754c 2024-12-04T19:16:15,226 DEBUG [StoreCloser-TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/8e390ba636864a1f946b620f3b447715 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/8e390ba636864a1f946b620f3b447715 2024-12-04T19:16:15,227 DEBUG [StoreCloser-TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/a0dbab56b1794da38ff032437238b981 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/a0dbab56b1794da38ff032437238b981 2024-12-04T19:16:15,228 DEBUG [StoreCloser-TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/5cd3638558d94d949c9f0420362bd353 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/5cd3638558d94d949c9f0420362bd353 2024-12-04T19:16:15,230 DEBUG [StoreCloser-TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/b73cb9348a444e5990f7dd68f3fa74e4 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/b73cb9348a444e5990f7dd68f3fa74e4 2024-12-04T19:16:15,231 DEBUG [StoreCloser-TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/2545b4e93ece49b1a2374a75a8787ce3 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/2545b4e93ece49b1a2374a75a8787ce3 2024-12-04T19:16:15,233 DEBUG [StoreCloser-TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/6def83e5daaa470cad924c91e8427da0, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/069acd4b31004462a3bf5807487fcbce, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/53e7d43d46674abebc52d9962cc4c76c, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/c3b895109f6d43d58f4530f89d377e12, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/ab4db85ae34a455591fd269cb6623d08, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/836a3d5aacae47de830f48f1faa051b5, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/930375700cc24b8799a269f65c50da26, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/ca43ec72a6c842ce911de40021f9b054, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/cda0c3296f8a4bf18b36f030e8a76ec1, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/6fea85c990d34e8ba39061661e359986, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/baa05d747946429a8756ac1bb3126219, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/698f6244952648838ae12a6981722d8a, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/bba9c182323a4afc9a919f4bcb72cafd, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/abe3860b56d0448c8e2a2d0066720db1, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/f827d30274234dc594221ccb0a525e9d, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/a4be3e0c72464e8292ac07b298300bcd, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/6204e0fa677c446c88d2c9674c024674, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/605ed11b9c484897b6e7f5a3d4537f65, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/b716959b1102447685288f4963b327cc, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/755633419de647048543c9f4bce94ab5, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/20010184066c47aba5acd7458944fb85, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/731e75cf8406410bae8db2a58f1601c7, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/4d98aac7ba314f2d9d4e397ee7d821c8, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/ae4e5d4ad7ff41118c44e0197ab725d0, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/abd0b586f3804270b14b1b3f03370180] to archive 2024-12-04T19:16:15,234 DEBUG [StoreCloser-TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-04T19:16:15,235 DEBUG [StoreCloser-TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/6def83e5daaa470cad924c91e8427da0 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/6def83e5daaa470cad924c91e8427da0 2024-12-04T19:16:15,236 DEBUG [StoreCloser-TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/069acd4b31004462a3bf5807487fcbce to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/069acd4b31004462a3bf5807487fcbce 2024-12-04T19:16:15,237 DEBUG [StoreCloser-TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/53e7d43d46674abebc52d9962cc4c76c to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/53e7d43d46674abebc52d9962cc4c76c 2024-12-04T19:16:15,238 DEBUG [StoreCloser-TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/c3b895109f6d43d58f4530f89d377e12 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/c3b895109f6d43d58f4530f89d377e12 2024-12-04T19:16:15,238 DEBUG [StoreCloser-TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/ab4db85ae34a455591fd269cb6623d08 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/ab4db85ae34a455591fd269cb6623d08 2024-12-04T19:16:15,239 DEBUG [StoreCloser-TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/836a3d5aacae47de830f48f1faa051b5 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/836a3d5aacae47de830f48f1faa051b5 2024-12-04T19:16:15,240 DEBUG [StoreCloser-TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/930375700cc24b8799a269f65c50da26 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/930375700cc24b8799a269f65c50da26 2024-12-04T19:16:15,241 DEBUG [StoreCloser-TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/ca43ec72a6c842ce911de40021f9b054 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/ca43ec72a6c842ce911de40021f9b054 2024-12-04T19:16:15,241 DEBUG [StoreCloser-TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/cda0c3296f8a4bf18b36f030e8a76ec1 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/cda0c3296f8a4bf18b36f030e8a76ec1 2024-12-04T19:16:15,242 DEBUG [StoreCloser-TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/6fea85c990d34e8ba39061661e359986 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/6fea85c990d34e8ba39061661e359986 2024-12-04T19:16:15,243 DEBUG [StoreCloser-TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/baa05d747946429a8756ac1bb3126219 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/baa05d747946429a8756ac1bb3126219 2024-12-04T19:16:15,243 DEBUG [StoreCloser-TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/698f6244952648838ae12a6981722d8a to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/698f6244952648838ae12a6981722d8a 2024-12-04T19:16:15,244 DEBUG [StoreCloser-TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/bba9c182323a4afc9a919f4bcb72cafd to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/bba9c182323a4afc9a919f4bcb72cafd 2024-12-04T19:16:15,245 DEBUG [StoreCloser-TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/abe3860b56d0448c8e2a2d0066720db1 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/abe3860b56d0448c8e2a2d0066720db1 2024-12-04T19:16:15,245 DEBUG [StoreCloser-TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/f827d30274234dc594221ccb0a525e9d to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/f827d30274234dc594221ccb0a525e9d 2024-12-04T19:16:15,246 DEBUG [StoreCloser-TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/a4be3e0c72464e8292ac07b298300bcd to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/a4be3e0c72464e8292ac07b298300bcd 2024-12-04T19:16:15,247 DEBUG [StoreCloser-TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/6204e0fa677c446c88d2c9674c024674 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/6204e0fa677c446c88d2c9674c024674 2024-12-04T19:16:15,247 DEBUG [StoreCloser-TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/605ed11b9c484897b6e7f5a3d4537f65 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/605ed11b9c484897b6e7f5a3d4537f65 2024-12-04T19:16:15,249 DEBUG [StoreCloser-TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/b716959b1102447685288f4963b327cc to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/b716959b1102447685288f4963b327cc 2024-12-04T19:16:15,249 DEBUG [StoreCloser-TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/755633419de647048543c9f4bce94ab5 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/755633419de647048543c9f4bce94ab5 2024-12-04T19:16:15,250 DEBUG [StoreCloser-TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/20010184066c47aba5acd7458944fb85 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/20010184066c47aba5acd7458944fb85 2024-12-04T19:16:15,251 DEBUG [StoreCloser-TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/731e75cf8406410bae8db2a58f1601c7 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/731e75cf8406410bae8db2a58f1601c7 2024-12-04T19:16:15,252 DEBUG [StoreCloser-TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/4d98aac7ba314f2d9d4e397ee7d821c8 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/4d98aac7ba314f2d9d4e397ee7d821c8 2024-12-04T19:16:15,253 DEBUG [StoreCloser-TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/ae4e5d4ad7ff41118c44e0197ab725d0 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/ae4e5d4ad7ff41118c44e0197ab725d0 2024-12-04T19:16:15,254 DEBUG [StoreCloser-TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/abd0b586f3804270b14b1b3f03370180 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/abd0b586f3804270b14b1b3f03370180 2024-12-04T19:16:15,258 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/recovered.edits/392.seqid, newMaxSeqId=392, maxSeqId=1 2024-12-04T19:16:15,258 INFO [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. 2024-12-04T19:16:15,259 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1676): Region close journal for 48c104a4c7a51d8b62d084d2a093043a: Waiting for close lock at 1733339773832Running coprocessor pre-close hooks at 1733339773832Disabling compacts and flushes for region at 1733339773832Disabling writes for close at 1733339773946 (+114 ms)Obtaining lock to block concurrent updates at 1733339773946Preparing flush snapshotting stores in 48c104a4c7a51d8b62d084d2a093043a at 1733339773946Finished memstore snapshotting TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1733339773946Flushing stores of TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a. at 1733339773946Flushing 48c104a4c7a51d8b62d084d2a093043a/A: creating writer at 1733339773946Flushing 48c104a4c7a51d8b62d084d2a093043a/A: appending metadata at 1733339773948 (+2 ms)Flushing 48c104a4c7a51d8b62d084d2a093043a/A: closing flushed file at 1733339773948Flushing 48c104a4c7a51d8b62d084d2a093043a/B: creating writer at 1733339774355 (+407 ms)Flushing 48c104a4c7a51d8b62d084d2a093043a/B: appending metadata at 1733339774356 (+1 ms)Flushing 48c104a4c7a51d8b62d084d2a093043a/B: closing flushed file at 1733339774356Flushing 48c104a4c7a51d8b62d084d2a093043a/C: creating writer at 1733339774766 (+410 ms)Flushing 48c104a4c7a51d8b62d084d2a093043a/C: appending metadata at 1733339774768 (+2 ms)Flushing 48c104a4c7a51d8b62d084d2a093043a/C: closing flushed file at 1733339774768Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@921d40f: reopening flushed file at 1733339775181 (+413 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2671150: reopening flushed file at 1733339775185 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7846c11e: reopening flushed file at 1733339775188 (+3 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 48c104a4c7a51d8b62d084d2a093043a in 1245ms, sequenceid=389, compaction requested=true at 1733339775191 (+3 ms)Writing region close event to WAL at 1733339775256 (+65 ms)Running coprocessor post-close hooks at 1733339775258 (+2 ms)Closed at 1733339775258 2024-12-04T19:16:15,261 INFO [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] handler.UnassignRegionHandler(157): Closed 48c104a4c7a51d8b62d084d2a093043a 2024-12-04T19:16:15,261 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=48c104a4c7a51d8b62d084d2a093043a, regionState=CLOSED 2024-12-04T19:16:15,262 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=138, ppid=137, state=RUNNABLE, hasLock=false; CloseRegionProcedure 48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 because future has completed 2024-12-04T19:16:15,264 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-12-04T19:16:15,264 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; CloseRegionProcedure 48c104a4c7a51d8b62d084d2a093043a, server=9ba2734cd39f,33969,1733339629791 in 1.5840 sec 2024-12-04T19:16:15,265 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=137, resume processing ppid=136 2024-12-04T19:16:15,265 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, ppid=136, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=48c104a4c7a51d8b62d084d2a093043a, UNASSIGN in 1.5890 sec 2024-12-04T19:16:15,267 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-12-04T19:16:15,267 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5930 sec 2024-12-04T19:16:15,268 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733339775267"}]},"ts":"1733339775267"} 2024-12-04T19:16:15,269 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-04T19:16:15,269 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-04T19:16:15,270 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.6030 sec 2024-12-04T19:16:15,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-04T19:16:15,822 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-04T19:16:15,823 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-04T19:16:15,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T19:16:15,826 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=139, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T19:16:15,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-12-04T19:16:15,828 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=139, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T19:16:15,849 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a 2024-12-04T19:16:15,857 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A, FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B, FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C, FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/recovered.edits] 2024-12-04T19:16:15,860 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/21c2540f5e6d4e35b85c7089a75df00c to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/21c2540f5e6d4e35b85c7089a75df00c 2024-12-04T19:16:15,863 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/65e880b2956945d69162cf031b8c4105 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/65e880b2956945d69162cf031b8c4105 2024-12-04T19:16:15,865 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/d1527bc8a7544041adbd8034297979ff to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/d1527bc8a7544041adbd8034297979ff 2024-12-04T19:16:15,867 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/f054f83c06834373b25421ecd37ca0fc to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/A/f054f83c06834373b25421ecd37ca0fc 2024-12-04T19:16:15,870 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/36b3643e067f471289f0966ab37d8fca to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/36b3643e067f471289f0966ab37d8fca 2024-12-04T19:16:15,871 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/a352dde772fa45d2bd45f911ff5c6926 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/a352dde772fa45d2bd45f911ff5c6926 2024-12-04T19:16:15,873 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/a91923fe33454a9093cacd03b2d71f59 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/a91923fe33454a9093cacd03b2d71f59 2024-12-04T19:16:15,874 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/affb887500934a77a994e827e9868ad7 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/B/affb887500934a77a994e827e9868ad7 2024-12-04T19:16:15,878 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/0b1c5b2b6700446b9e5eadbe3598b9e8 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/0b1c5b2b6700446b9e5eadbe3598b9e8 2024-12-04T19:16:15,879 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/160324cf34dd44d7842ce348da97be0e to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/160324cf34dd44d7842ce348da97be0e 2024-12-04T19:16:15,880 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/48567464762b4ff28caaccce0629579f to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/48567464762b4ff28caaccce0629579f 2024-12-04T19:16:15,882 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/6fdff6a5b77d4ee5acd0bbe6d6e841fe to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/C/6fdff6a5b77d4ee5acd0bbe6d6e841fe 2024-12-04T19:16:15,884 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/recovered.edits/392.seqid to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a/recovered.edits/392.seqid 2024-12-04T19:16:15,885 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/48c104a4c7a51d8b62d084d2a093043a 2024-12-04T19:16:15,885 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-04T19:16:15,888 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=139, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T19:16:15,890 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-04T19:16:15,893 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-04T19:16:15,894 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=139, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T19:16:15,894 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-04T19:16:15,895 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733339775894"}]},"ts":"9223372036854775807"} 2024-12-04T19:16:15,899 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-04T19:16:15,899 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 48c104a4c7a51d8b62d084d2a093043a, NAME => 'TestAcidGuarantees,,1733339750863.48c104a4c7a51d8b62d084d2a093043a.', STARTKEY => '', ENDKEY => ''}] 2024-12-04T19:16:15,899 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-04T19:16:15,899 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733339775899"}]},"ts":"9223372036854775807"} 2024-12-04T19:16:15,901 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-04T19:16:15,902 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=139, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T19:16:15,904 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 79 msec 2024-12-04T19:16:15,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-12-04T19:16:15,934 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-04T19:16:15,937 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-04T19:16:15,946 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=250 (was 251), OpenFileDescriptor=447 (was 461), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=471 (was 440) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=15134 (was 15613) 2024-12-04T19:16:15,955 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=250, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=471, ProcessCount=11, AvailableMemoryMB=15133 2024-12-04T19:16:15,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-04T19:16:15,957 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T19:16:15,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] procedure2.ProcedureExecutor(1139): Stored pid=140, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-04T19:16:15,959 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=140, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-04T19:16:15,959 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:15,959 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 140 2024-12-04T19:16:15,960 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=140, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-04T19:16:15,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-12-04T19:16:15,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742359_1535 (size=1061) 2024-12-04T19:16:15,970 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d 2024-12-04T19:16:15,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742360_1536 (size=53) 2024-12-04T19:16:15,982 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T19:16:15,982 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing e22a2b2fd059f99265758fd39ea40832, disabling compactions & flushes 2024-12-04T19:16:15,982 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. 2024-12-04T19:16:15,982 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. 2024-12-04T19:16:15,982 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. after waiting 0 ms 2024-12-04T19:16:15,982 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. 2024-12-04T19:16:15,982 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. 2024-12-04T19:16:15,983 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for e22a2b2fd059f99265758fd39ea40832: Waiting for close lock at 1733339775982Disabling compacts and flushes for region at 1733339775982Disabling writes for close at 1733339775982Writing region close event to WAL at 1733339775982Closed at 1733339775982 2024-12-04T19:16:15,984 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=140, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-04T19:16:15,984 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733339775984"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733339775984"}]},"ts":"1733339775984"} 2024-12-04T19:16:15,986 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-04T19:16:15,987 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=140, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-04T19:16:15,987 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733339775987"}]},"ts":"1733339775987"} 2024-12-04T19:16:15,992 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-04T19:16:15,993 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e22a2b2fd059f99265758fd39ea40832, ASSIGN}] 2024-12-04T19:16:15,994 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e22a2b2fd059f99265758fd39ea40832, ASSIGN 2024-12-04T19:16:15,995 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e22a2b2fd059f99265758fd39ea40832, ASSIGN; state=OFFLINE, location=9ba2734cd39f,33969,1733339629791; forceNewPlan=false, retain=false 2024-12-04T19:16:16,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-12-04T19:16:16,146 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=e22a2b2fd059f99265758fd39ea40832, regionState=OPENING, regionLocation=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:16,148 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e22a2b2fd059f99265758fd39ea40832, ASSIGN because future has completed 2024-12-04T19:16:16,148 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE, hasLock=false; OpenRegionProcedure e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791}] 2024-12-04T19:16:16,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-12-04T19:16:16,302 INFO [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. 2024-12-04T19:16:16,303 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(7752): Opening region: {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} 2024-12-04T19:16:16,303 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees e22a2b2fd059f99265758fd39ea40832 2024-12-04T19:16:16,303 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T19:16:16,303 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(7794): checking encryption for e22a2b2fd059f99265758fd39ea40832 2024-12-04T19:16:16,303 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(7797): checking classloading for e22a2b2fd059f99265758fd39ea40832 2024-12-04T19:16:16,306 INFO [StoreOpener-e22a2b2fd059f99265758fd39ea40832-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region e22a2b2fd059f99265758fd39ea40832 2024-12-04T19:16:16,307 INFO [StoreOpener-e22a2b2fd059f99265758fd39ea40832-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-04T19:16:16,307 INFO [StoreOpener-e22a2b2fd059f99265758fd39ea40832-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e22a2b2fd059f99265758fd39ea40832 columnFamilyName A 2024-12-04T19:16:16,307 DEBUG [StoreOpener-e22a2b2fd059f99265758fd39ea40832-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:16,308 INFO [StoreOpener-e22a2b2fd059f99265758fd39ea40832-1 {}] regionserver.HStore(327): Store=e22a2b2fd059f99265758fd39ea40832/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:16:16,308 INFO [StoreOpener-e22a2b2fd059f99265758fd39ea40832-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region e22a2b2fd059f99265758fd39ea40832 2024-12-04T19:16:16,309 INFO [StoreOpener-e22a2b2fd059f99265758fd39ea40832-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-04T19:16:16,309 INFO [StoreOpener-e22a2b2fd059f99265758fd39ea40832-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e22a2b2fd059f99265758fd39ea40832 columnFamilyName B 2024-12-04T19:16:16,309 DEBUG [StoreOpener-e22a2b2fd059f99265758fd39ea40832-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:16,309 INFO [StoreOpener-e22a2b2fd059f99265758fd39ea40832-1 {}] regionserver.HStore(327): Store=e22a2b2fd059f99265758fd39ea40832/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:16:16,309 INFO [StoreOpener-e22a2b2fd059f99265758fd39ea40832-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region e22a2b2fd059f99265758fd39ea40832 2024-12-04T19:16:16,310 INFO [StoreOpener-e22a2b2fd059f99265758fd39ea40832-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-04T19:16:16,311 INFO [StoreOpener-e22a2b2fd059f99265758fd39ea40832-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e22a2b2fd059f99265758fd39ea40832 columnFamilyName C 2024-12-04T19:16:16,311 DEBUG [StoreOpener-e22a2b2fd059f99265758fd39ea40832-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:16,311 INFO [StoreOpener-e22a2b2fd059f99265758fd39ea40832-1 {}] regionserver.HStore(327): Store=e22a2b2fd059f99265758fd39ea40832/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:16:16,311 INFO [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. 2024-12-04T19:16:16,311 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1038): replaying wal for e22a2b2fd059f99265758fd39ea40832 2024-12-04T19:16:16,312 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832 2024-12-04T19:16:16,312 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832 2024-12-04T19:16:16,313 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1048): stopping wal replay for e22a2b2fd059f99265758fd39ea40832 2024-12-04T19:16:16,313 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1060): Cleaning up temporary data for e22a2b2fd059f99265758fd39ea40832 2024-12-04T19:16:16,313 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-04T19:16:16,314 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1093): writing seq id for e22a2b2fd059f99265758fd39ea40832 2024-12-04T19:16:16,316 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T19:16:16,316 INFO [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1114): Opened e22a2b2fd059f99265758fd39ea40832; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62694890, jitterRate=-0.06577333807945251}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-04T19:16:16,316 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e22a2b2fd059f99265758fd39ea40832 2024-12-04T19:16:16,317 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1006): Region open journal for e22a2b2fd059f99265758fd39ea40832: Running coprocessor pre-open hook at 1733339776303Writing region info on filesystem at 1733339776303Initializing all the Stores at 1733339776306 (+3 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733339776306Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733339776306Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733339776306Cleaning up temporary data from old regions at 1733339776313 (+7 ms)Running coprocessor post-open hooks at 1733339776316 (+3 ms)Region opened successfully at 1733339776317 (+1 ms) 2024-12-04T19:16:16,318 INFO [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., pid=142, masterSystemTime=1733339776300 2024-12-04T19:16:16,320 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. 2024-12-04T19:16:16,320 INFO [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. 2024-12-04T19:16:16,324 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=e22a2b2fd059f99265758fd39ea40832, regionState=OPEN, openSeqNum=2, regionLocation=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:16,326 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=142, ppid=141, state=RUNNABLE, hasLock=false; OpenRegionProcedure e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 because future has completed 2024-12-04T19:16:16,328 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-12-04T19:16:16,329 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; OpenRegionProcedure e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 in 179 msec 2024-12-04T19:16:16,331 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=141, resume processing ppid=140 2024-12-04T19:16:16,331 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, ppid=140, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e22a2b2fd059f99265758fd39ea40832, ASSIGN in 336 msec 2024-12-04T19:16:16,332 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=140, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-04T19:16:16,332 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733339776332"}]},"ts":"1733339776332"} 2024-12-04T19:16:16,333 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-04T19:16:16,334 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=140, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-04T19:16:16,335 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 377 msec 2024-12-04T19:16:16,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-12-04T19:16:16,592 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-04T19:16:16,593 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37eedb90, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:16:16,594 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ba2734cd39f,46667,-1 for getting cluster id 2024-12-04T19:16:16,594 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T19:16:16,595 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9baaea35-8dd3-4653-9b7c-03e5b792e644' 2024-12-04T19:16:16,595 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T19:16:16,595 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9baaea35-8dd3-4653-9b7c-03e5b792e644" 2024-12-04T19:16:16,595 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2307099d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:16:16,595 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ba2734cd39f,46667,-1] 2024-12-04T19:16:16,596 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T19:16:16,596 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:16:16,596 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42844, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T19:16:16,597 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27d9b5c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:16:16,597 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T19:16:16,598 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ba2734cd39f,33969,1733339629791, seqNum=-1] 2024-12-04T19:16:16,599 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T19:16:16,599 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38870, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T19:16:16,600 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-04T19:16:16,601 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 9ba2734cd39f,46667,1733339628877 2024-12-04T19:16:16,601 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@bebf3c0 2024-12-04T19:16:16,601 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-04T19:16:16,602 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42850, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-04T19:16:16,604 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-04T19:16:16,604 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T19:16:16,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] procedure2.ProcedureExecutor(1139): Stored pid=143, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-04T19:16:16,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742361_1537 (size=1097) 2024-12-04T19:16:16,627 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-04T19:16:16,627 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-04T19:16:16,630 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-04T19:16:16,634 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e22a2b2fd059f99265758fd39ea40832, REOPEN/MOVE}] 2024-12-04T19:16:16,635 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e22a2b2fd059f99265758fd39ea40832, REOPEN/MOVE 2024-12-04T19:16:16,636 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=e22a2b2fd059f99265758fd39ea40832, regionState=CLOSING, regionLocation=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:16,638 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e22a2b2fd059f99265758fd39ea40832, REOPEN/MOVE because future has completed 2024-12-04T19:16:16,638 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-04T19:16:16,638 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE, hasLock=false; CloseRegionProcedure e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791}] 2024-12-04T19:16:16,792 INFO [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] handler.UnassignRegionHandler(122): Close e22a2b2fd059f99265758fd39ea40832 2024-12-04T19:16:16,792 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-04T19:16:16,792 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1722): Closing e22a2b2fd059f99265758fd39ea40832, disabling compactions & flushes 2024-12-04T19:16:16,792 INFO [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. 2024-12-04T19:16:16,792 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. 2024-12-04T19:16:16,792 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. after waiting 0 ms 2024-12-04T19:16:16,792 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. 2024-12-04T19:16:16,803 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-04T19:16:16,804 INFO [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. 2024-12-04T19:16:16,804 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1676): Region close journal for e22a2b2fd059f99265758fd39ea40832: Waiting for close lock at 1733339776792Running coprocessor pre-close hooks at 1733339776792Disabling compacts and flushes for region at 1733339776792Disabling writes for close at 1733339776792Writing region close event to WAL at 1733339776794 (+2 ms)Running coprocessor post-close hooks at 1733339776804 (+10 ms)Closed at 1733339776804 2024-12-04T19:16:16,804 WARN [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegionServer(3299): Not adding moved region record: e22a2b2fd059f99265758fd39ea40832 to self. 2024-12-04T19:16:16,807 INFO [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] handler.UnassignRegionHandler(157): Closed e22a2b2fd059f99265758fd39ea40832 2024-12-04T19:16:16,807 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=e22a2b2fd059f99265758fd39ea40832, regionState=CLOSED 2024-12-04T19:16:16,810 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=146, ppid=145, state=RUNNABLE, hasLock=false; CloseRegionProcedure e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 because future has completed 2024-12-04T19:16:16,815 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=146, resume processing ppid=145 2024-12-04T19:16:16,815 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, ppid=145, state=SUCCESS, hasLock=false; CloseRegionProcedure e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 in 173 msec 2024-12-04T19:16:16,817 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e22a2b2fd059f99265758fd39ea40832, REOPEN/MOVE; state=CLOSED, location=9ba2734cd39f,33969,1733339629791; forceNewPlan=false, retain=true 2024-12-04T19:16:16,968 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=e22a2b2fd059f99265758fd39ea40832, regionState=OPENING, regionLocation=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:16,970 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e22a2b2fd059f99265758fd39ea40832, REOPEN/MOVE because future has completed 2024-12-04T19:16:16,970 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=145, state=RUNNABLE, hasLock=false; OpenRegionProcedure e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791}] 2024-12-04T19:16:17,126 INFO [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. 2024-12-04T19:16:17,126 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(7752): Opening region: {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} 2024-12-04T19:16:17,127 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees e22a2b2fd059f99265758fd39ea40832 2024-12-04T19:16:17,127 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T19:16:17,127 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(7794): checking encryption for e22a2b2fd059f99265758fd39ea40832 2024-12-04T19:16:17,127 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(7797): checking classloading for e22a2b2fd059f99265758fd39ea40832 2024-12-04T19:16:17,128 INFO [StoreOpener-e22a2b2fd059f99265758fd39ea40832-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region e22a2b2fd059f99265758fd39ea40832 2024-12-04T19:16:17,128 INFO [StoreOpener-e22a2b2fd059f99265758fd39ea40832-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-04T19:16:17,129 INFO [StoreOpener-e22a2b2fd059f99265758fd39ea40832-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e22a2b2fd059f99265758fd39ea40832 columnFamilyName A 2024-12-04T19:16:17,129 DEBUG [StoreOpener-e22a2b2fd059f99265758fd39ea40832-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:17,133 INFO [StoreOpener-e22a2b2fd059f99265758fd39ea40832-1 {}] regionserver.HStore(327): Store=e22a2b2fd059f99265758fd39ea40832/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:16:17,133 INFO [StoreOpener-e22a2b2fd059f99265758fd39ea40832-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region e22a2b2fd059f99265758fd39ea40832 2024-12-04T19:16:17,134 INFO [StoreOpener-e22a2b2fd059f99265758fd39ea40832-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-04T19:16:17,134 INFO [StoreOpener-e22a2b2fd059f99265758fd39ea40832-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e22a2b2fd059f99265758fd39ea40832 columnFamilyName B 2024-12-04T19:16:17,134 DEBUG [StoreOpener-e22a2b2fd059f99265758fd39ea40832-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:17,135 INFO [StoreOpener-e22a2b2fd059f99265758fd39ea40832-1 {}] regionserver.HStore(327): Store=e22a2b2fd059f99265758fd39ea40832/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:16:17,135 INFO [StoreOpener-e22a2b2fd059f99265758fd39ea40832-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region e22a2b2fd059f99265758fd39ea40832 2024-12-04T19:16:17,135 INFO [StoreOpener-e22a2b2fd059f99265758fd39ea40832-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-04T19:16:17,135 INFO [StoreOpener-e22a2b2fd059f99265758fd39ea40832-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e22a2b2fd059f99265758fd39ea40832 columnFamilyName C 2024-12-04T19:16:17,135 DEBUG [StoreOpener-e22a2b2fd059f99265758fd39ea40832-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:17,136 INFO [StoreOpener-e22a2b2fd059f99265758fd39ea40832-1 {}] regionserver.HStore(327): Store=e22a2b2fd059f99265758fd39ea40832/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:16:17,136 INFO [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. 2024-12-04T19:16:17,136 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1038): replaying wal for e22a2b2fd059f99265758fd39ea40832 2024-12-04T19:16:17,139 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832 2024-12-04T19:16:17,140 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832 2024-12-04T19:16:17,141 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1048): stopping wal replay for e22a2b2fd059f99265758fd39ea40832 2024-12-04T19:16:17,141 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1060): Cleaning up temporary data for e22a2b2fd059f99265758fd39ea40832 2024-12-04T19:16:17,141 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-04T19:16:17,143 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1093): writing seq id for e22a2b2fd059f99265758fd39ea40832 2024-12-04T19:16:17,143 INFO [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1114): Opened e22a2b2fd059f99265758fd39ea40832; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60419670, jitterRate=-0.09967675805091858}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-04T19:16:17,144 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e22a2b2fd059f99265758fd39ea40832 2024-12-04T19:16:17,144 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1006): Region open journal for e22a2b2fd059f99265758fd39ea40832: Running coprocessor pre-open hook at 1733339777127Writing region info on filesystem at 1733339777127Initializing all the Stores at 1733339777127Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733339777127Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733339777128 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733339777128Cleaning up temporary data from old regions at 1733339777141 (+13 ms)Running coprocessor post-open hooks at 1733339777144 (+3 ms)Region opened successfully at 1733339777144 2024-12-04T19:16:17,145 INFO [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., pid=147, masterSystemTime=1733339777123 2024-12-04T19:16:17,147 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. 2024-12-04T19:16:17,147 INFO [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. 2024-12-04T19:16:17,148 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=e22a2b2fd059f99265758fd39ea40832, regionState=OPEN, openSeqNum=5, regionLocation=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:17,149 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=147, ppid=145, state=RUNNABLE, hasLock=false; OpenRegionProcedure e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 because future has completed 2024-12-04T19:16:17,153 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=145 2024-12-04T19:16:17,153 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=145, state=SUCCESS, hasLock=false; OpenRegionProcedure e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 in 180 msec 2024-12-04T19:16:17,155 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=144 2024-12-04T19:16:17,156 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=144, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e22a2b2fd059f99265758fd39ea40832, REOPEN/MOVE in 519 msec 2024-12-04T19:16:17,158 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-12-04T19:16:17,158 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 526 msec 2024-12-04T19:16:17,160 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 553 msec 2024-12-04T19:16:17,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-12-04T19:16:17,161 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-04T19:16:17,162 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17bc30a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:16:17,162 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ba2734cd39f,46667,-1 for getting cluster id 2024-12-04T19:16:17,162 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T19:16:17,163 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9baaea35-8dd3-4653-9b7c-03e5b792e644' 2024-12-04T19:16:17,163 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T19:16:17,164 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9baaea35-8dd3-4653-9b7c-03e5b792e644" 2024-12-04T19:16:17,164 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@502aadd1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:16:17,164 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ba2734cd39f,46667,-1] 2024-12-04T19:16:17,164 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T19:16:17,164 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:16:17,165 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42876, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T19:16:17,165 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7bf516c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:16:17,166 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@227ae0e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:16:17,166 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ba2734cd39f,46667,-1 for getting cluster id 2024-12-04T19:16:17,166 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T19:16:17,167 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9baaea35-8dd3-4653-9b7c-03e5b792e644' 2024-12-04T19:16:17,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T19:16:17,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9baaea35-8dd3-4653-9b7c-03e5b792e644" 2024-12-04T19:16:17,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@769025aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:16:17,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ba2734cd39f,46667,-1] 2024-12-04T19:16:17,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T19:16:17,168 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:16:17,168 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42890, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T19:16:17,169 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43d21dc0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:16:17,169 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@401e798, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:16:17,169 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ba2734cd39f,46667,-1 for getting cluster id 2024-12-04T19:16:17,169 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T19:16:17,170 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9baaea35-8dd3-4653-9b7c-03e5b792e644' 2024-12-04T19:16:17,170 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T19:16:17,170 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9baaea35-8dd3-4653-9b7c-03e5b792e644" 2024-12-04T19:16:17,170 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25bcb7e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:16:17,170 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ba2734cd39f,46667,-1] 2024-12-04T19:16:17,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T19:16:17,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:16:17,171 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42902, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T19:16:17,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60853c9b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:16:17,173 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a2ee11c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:16:17,173 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ba2734cd39f,46667,-1 for getting cluster id 2024-12-04T19:16:17,173 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T19:16:17,174 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9baaea35-8dd3-4653-9b7c-03e5b792e644' 2024-12-04T19:16:17,174 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T19:16:17,174 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9baaea35-8dd3-4653-9b7c-03e5b792e644" 2024-12-04T19:16:17,175 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ec0a0cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:16:17,175 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ba2734cd39f,46667,-1] 2024-12-04T19:16:17,175 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T19:16:17,175 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:16:17,176 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42924, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T19:16:17,176 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d8dcb81, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:16:17,177 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53cbdd75, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:16:17,177 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ba2734cd39f,46667,-1 for getting cluster id 2024-12-04T19:16:17,178 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T19:16:17,178 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9baaea35-8dd3-4653-9b7c-03e5b792e644' 2024-12-04T19:16:17,178 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T19:16:17,179 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9baaea35-8dd3-4653-9b7c-03e5b792e644" 2024-12-04T19:16:17,179 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5013accc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:16:17,179 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ba2734cd39f,46667,-1] 2024-12-04T19:16:17,179 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T19:16:17,180 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:16:17,180 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42942, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T19:16:17,181 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20896981, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:16:17,182 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1001aa8b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:16:17,182 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ba2734cd39f,46667,-1 for getting cluster id 2024-12-04T19:16:17,182 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T19:16:17,183 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9baaea35-8dd3-4653-9b7c-03e5b792e644' 2024-12-04T19:16:17,183 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T19:16:17,183 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9baaea35-8dd3-4653-9b7c-03e5b792e644" 2024-12-04T19:16:17,183 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68c36b4d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:16:17,183 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ba2734cd39f,46667,-1] 2024-12-04T19:16:17,183 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T19:16:17,184 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:16:17,185 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42968, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T19:16:17,186 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31617aa3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:16:17,188 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a7bff21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:16:17,189 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ba2734cd39f,46667,-1 for getting cluster id 2024-12-04T19:16:17,189 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T19:16:17,190 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9baaea35-8dd3-4653-9b7c-03e5b792e644' 2024-12-04T19:16:17,190 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T19:16:17,190 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9baaea35-8dd3-4653-9b7c-03e5b792e644" 2024-12-04T19:16:17,190 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74cce6e5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:16:17,190 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ba2734cd39f,46667,-1] 2024-12-04T19:16:17,191 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T19:16:17,191 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:16:17,192 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42986, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T19:16:17,192 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fcc8803, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:16:17,193 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2daf9160, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:16:17,193 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ba2734cd39f,46667,-1 for getting cluster id 2024-12-04T19:16:17,193 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T19:16:17,194 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9baaea35-8dd3-4653-9b7c-03e5b792e644' 2024-12-04T19:16:17,195 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T19:16:17,195 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9baaea35-8dd3-4653-9b7c-03e5b792e644" 2024-12-04T19:16:17,195 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3af9d557, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:16:17,195 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ba2734cd39f,46667,-1] 2024-12-04T19:16:17,195 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T19:16:17,195 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:16:17,196 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43010, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T19:16:17,197 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57cf9ed4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:16:17,198 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6719bbfd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:16:17,198 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ba2734cd39f,46667,-1 for getting cluster id 2024-12-04T19:16:17,198 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T19:16:17,199 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9baaea35-8dd3-4653-9b7c-03e5b792e644' 2024-12-04T19:16:17,199 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T19:16:17,199 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9baaea35-8dd3-4653-9b7c-03e5b792e644" 2024-12-04T19:16:17,199 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20632211, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:16:17,199 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ba2734cd39f,46667,-1] 2024-12-04T19:16:17,199 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T19:16:17,200 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:16:17,200 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43016, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T19:16:17,201 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c320798, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:16:17,204 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@611845f8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:16:17,204 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ba2734cd39f,46667,-1 for getting cluster id 2024-12-04T19:16:17,205 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T19:16:17,206 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9baaea35-8dd3-4653-9b7c-03e5b792e644' 2024-12-04T19:16:17,206 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T19:16:17,206 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9baaea35-8dd3-4653-9b7c-03e5b792e644" 2024-12-04T19:16:17,206 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6344ace3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:16:17,206 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ba2734cd39f,46667,-1] 2024-12-04T19:16:17,207 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T19:16:17,207 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:16:17,208 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43022, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T19:16:17,208 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@740e9bdd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:16:17,209 DEBUG [Thread-2402 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T19:16:17,209 DEBUG [Thread-2393 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T19:16:17,209 DEBUG [Thread-2406 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T19:16:17,209 DEBUG [Thread-2397 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T19:16:17,210 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ba2734cd39f,33969,1733339629791, seqNum=-1] 2024-12-04T19:16:17,210 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ba2734cd39f,33969,1733339629791, seqNum=-1] 2024-12-04T19:16:17,210 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ba2734cd39f,33969,1733339629791, seqNum=-1] 2024-12-04T19:16:17,210 DEBUG [Thread-2395 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T19:16:17,210 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T19:16:17,210 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T19:16:17,210 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T19:16:17,210 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ba2734cd39f,33969,1733339629791, seqNum=-1] 2024-12-04T19:16:17,210 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T19:16:17,211 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ba2734cd39f,33969,1733339629791, seqNum=-1] 2024-12-04T19:16:17,211 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T19:16:17,212 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38904, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T19:16:17,212 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38884, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T19:16:17,212 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38908, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T19:16:17,213 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38914, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T19:16:17,213 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38886, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T19:16:17,214 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5] 2024-12-04T19:16:17,215 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5] 2024-12-04T19:16:17,216 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5] 2024-12-04T19:16:17,216 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5] 2024-12-04T19:16:17,218 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5] 2024-12-04T19:16:17,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(8855): Flush requested on e22a2b2fd059f99265758fd39ea40832 2024-12-04T19:16:17,225 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e22a2b2fd059f99265758fd39ea40832 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-04T19:16:17,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e22a2b2fd059f99265758fd39ea40832, store=A 2024-12-04T19:16:17,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:16:17,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e22a2b2fd059f99265758fd39ea40832, store=B 2024-12-04T19:16:17,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:16:17,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e22a2b2fd059f99265758fd39ea40832, store=C 2024-12-04T19:16:17,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:16:17,235 DEBUG [Thread-2391 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T19:16:17,236 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ba2734cd39f,33969,1733339629791, seqNum=-1] 2024-12-04T19:16:17,236 DEBUG [Thread-2410 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T19:16:17,236 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T19:16:17,237 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ba2734cd39f,33969,1733339629791, seqNum=-1] 2024-12-04T19:16:17,237 DEBUG [Thread-2404 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T19:16:17,237 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T19:16:17,238 DEBUG [Thread-2404 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ba2734cd39f,33969,1733339629791, seqNum=-1] 2024-12-04T19:16:17,238 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38922, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T19:16:17,238 DEBUG [Thread-2404 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T19:16:17,239 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38938, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T19:16:17,239 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5] 2024-12-04T19:16:17,240 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38924, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T19:16:17,242 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5] 2024-12-04T19:16:17,243 DEBUG [Thread-2399 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T19:16:17,243 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T19:16:17,243 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ba2734cd39f,33969,1733339629791, seqNum=-1] 2024-12-04T19:16:17,244 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T19:16:17,244 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5] 2024-12-04T19:16:17,245 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38946, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T19:16:17,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] procedure2.ProcedureExecutor(1139): Stored pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees 2024-12-04T19:16:17,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-04T19:16:17,247 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5] 2024-12-04T19:16:17,249 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T19:16:17,250 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T19:16:17,250 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T19:16:17,255 DEBUG [Thread-2408 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T19:16:17,256 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ba2734cd39f,33969,1733339629791, seqNum=-1] 2024-12-04T19:16:17,256 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T19:16:17,257 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:17,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38922 deadline: 1733339837251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:17,257 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38952, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T19:16:17,258 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:17,258 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:17,258 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:17,258 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:17,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38914 deadline: 1733339837252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:17,258 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:17,258 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:17,258 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:17,258 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5] 2024-12-04T19:16:17,261 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:17,261 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:17,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38908 deadline: 1733339837257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:17,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38904 deadline: 1733339837257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:17,261 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:17,261 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:17,261 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:17,262 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:17,262 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:17,262 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:17,262 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204e2374b0a89904335995a74aa874b4fce_e22a2b2fd059f99265758fd39ea40832 is 50, key is test_row_0/A:col10/1733339777224/Put/seqid=0 2024-12-04T19:16:17,274 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:17,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 2 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38946 deadline: 1733339837272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:17,274 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:17,274 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:17,274 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:17,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742362_1538 (size=12154) 2024-12-04T19:16:17,311 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:17,317 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204e2374b0a89904335995a74aa874b4fce_e22a2b2fd059f99265758fd39ea40832 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204e2374b0a89904335995a74aa874b4fce_e22a2b2fd059f99265758fd39ea40832 2024-12-04T19:16:17,318 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/A/e9840326039a48a186a612cb083952c7, store: [table=TestAcidGuarantees family=A region=e22a2b2fd059f99265758fd39ea40832] 2024-12-04T19:16:17,318 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/A/e9840326039a48a186a612cb083952c7 is 175, key is test_row_0/A:col10/1733339777224/Put/seqid=0 2024-12-04T19:16:17,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-04T19:16:17,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742363_1539 (size=30955) 2024-12-04T19:16:17,364 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:17,364 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:17,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38922 deadline: 1733339837362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:17,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38914 deadline: 1733339837362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:17,364 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:17,364 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:17,364 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:17,364 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:17,364 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:17,364 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:17,373 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:17,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38904 deadline: 1733339837371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:17,373 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:17,373 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:17,373 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:17,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38908 deadline: 1733339837371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:17,373 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:17,373 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:17,373 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:17,373 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:17,384 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:17,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38946 deadline: 1733339837382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:17,384 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:17,384 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:17,384 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:17,402 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-04T19:16:17,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. 2024-12-04T19:16:17,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. as already flushing 2024-12-04T19:16:17,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. 2024-12-04T19:16:17,403 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:17,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:17,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:17,556 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-04T19:16:17,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. 2024-12-04T19:16:17,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. as already flushing 2024-12-04T19:16:17,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. 2024-12-04T19:16:17,556 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:17,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:17,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:17,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-04T19:16:17,572 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:17,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38914 deadline: 1733339837572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:17,573 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:17,573 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:17,573 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:17,574 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:17,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38922 deadline: 1733339837572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:17,574 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:17,574 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:17,574 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:17,584 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:17,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38904 deadline: 1733339837582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:17,584 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:17,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38908 deadline: 1733339837582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:17,584 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:17,584 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:17,585 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:17,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:17,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:17,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:17,594 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:17,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38946 deadline: 1733339837591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:17,594 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:17,594 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:17,594 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:17,710 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-04T19:16:17,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. 2024-12-04T19:16:17,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. as already flushing 2024-12-04T19:16:17,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. 2024-12-04T19:16:17,710 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:17,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:17,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:17,754 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/A/e9840326039a48a186a612cb083952c7 2024-12-04T19:16:17,788 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/B/fc097f27a101452eba3408fa6e892e3d is 50, key is test_row_0/B:col10/1733339777224/Put/seqid=0 2024-12-04T19:16:17,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742364_1540 (size=12001) 2024-12-04T19:16:17,826 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/B/fc097f27a101452eba3408fa6e892e3d 2024-12-04T19:16:17,858 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/C/79ccd2ff28b54a96842ea98b8af18281 is 50, key is test_row_0/C:col10/1733339777224/Put/seqid=0 2024-12-04T19:16:17,862 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-04T19:16:17,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. 2024-12-04T19:16:17,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. as already flushing 2024-12-04T19:16:17,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. 2024-12-04T19:16:17,863 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:17,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:17,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:17,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-04T19:16:17,885 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:17,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38914 deadline: 1733339837881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:17,886 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:17,886 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:17,886 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:17,886 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:17,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38922 deadline: 1733339837881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:17,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742365_1541 (size=12001) 2024-12-04T19:16:17,886 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:17,886 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:17,886 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:17,888 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/C/79ccd2ff28b54a96842ea98b8af18281 2024-12-04T19:16:17,895 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:17,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38904 deadline: 1733339837892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:17,896 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:17,896 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:17,896 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:17,896 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/A/e9840326039a48a186a612cb083952c7 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/e9840326039a48a186a612cb083952c7 2024-12-04T19:16:17,902 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/e9840326039a48a186a612cb083952c7, entries=150, sequenceid=16, filesize=30.2 K 2024-12-04T19:16:17,903 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/B/fc097f27a101452eba3408fa6e892e3d as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/fc097f27a101452eba3408fa6e892e3d 2024-12-04T19:16:17,905 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:17,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38908 deadline: 1733339837892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:17,909 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:17,910 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:17,910 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:17,910 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:17,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38946 deadline: 1733339837902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:17,910 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:17,910 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:17,910 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:17,918 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/fc097f27a101452eba3408fa6e892e3d, entries=150, sequenceid=16, filesize=11.7 K 2024-12-04T19:16:17,920 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/C/79ccd2ff28b54a96842ea98b8af18281 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/79ccd2ff28b54a96842ea98b8af18281 2024-12-04T19:16:17,931 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/79ccd2ff28b54a96842ea98b8af18281, entries=150, sequenceid=16, filesize=11.7 K 2024-12-04T19:16:17,932 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for e22a2b2fd059f99265758fd39ea40832 in 708ms, sequenceid=16, compaction requested=false 2024-12-04T19:16:17,932 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e22a2b2fd059f99265758fd39ea40832: 2024-12-04T19:16:18,014 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-04T19:16:18,018 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-04T19:16:18,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. 2024-12-04T19:16:18,018 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2902): Flushing e22a2b2fd059f99265758fd39ea40832 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-04T19:16:18,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e22a2b2fd059f99265758fd39ea40832, store=A 2024-12-04T19:16:18,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:16:18,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e22a2b2fd059f99265758fd39ea40832, store=B 2024-12-04T19:16:18,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:16:18,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e22a2b2fd059f99265758fd39ea40832, store=C 2024-12-04T19:16:18,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:16:18,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120447347198501c4dc1b69339cfb7a94899_e22a2b2fd059f99265758fd39ea40832 is 50, key is test_row_0/A:col10/1733339777255/Put/seqid=0 2024-12-04T19:16:18,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742366_1542 (size=12154) 2024-12-04T19:16:18,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:18,054 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120447347198501c4dc1b69339cfb7a94899_e22a2b2fd059f99265758fd39ea40832 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120447347198501c4dc1b69339cfb7a94899_e22a2b2fd059f99265758fd39ea40832 2024-12-04T19:16:18,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/A/6e8ada5fc5614ae6a4dea92a5a8ddd99, store: [table=TestAcidGuarantees family=A region=e22a2b2fd059f99265758fd39ea40832] 2024-12-04T19:16:18,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/A/6e8ada5fc5614ae6a4dea92a5a8ddd99 is 175, key is test_row_0/A:col10/1733339777255/Put/seqid=0 2024-12-04T19:16:18,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742367_1543 (size=30955) 2024-12-04T19:16:18,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-04T19:16:18,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(8855): Flush requested on e22a2b2fd059f99265758fd39ea40832 2024-12-04T19:16:18,393 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. as already flushing 2024-12-04T19:16:18,411 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:18,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38914 deadline: 1733339838409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:18,412 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:18,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38922 deadline: 1733339838409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:18,412 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:18,412 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:18,412 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:18,413 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:18,413 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:18,413 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:18,413 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:18,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38904 deadline: 1733339838410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:18,413 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:18,413 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:18,414 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:18,422 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:18,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38908 deadline: 1733339838421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:18,423 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:18,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:18,423 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:18,423 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:18,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38946 deadline: 1733339838421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:18,423 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:18,423 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:18,423 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:18,475 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=36, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/A/6e8ada5fc5614ae6a4dea92a5a8ddd99 2024-12-04T19:16:18,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/B/960c8e993e7b47bea1e536660db46177 is 50, key is test_row_0/B:col10/1733339777255/Put/seqid=0 2024-12-04T19:16:18,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742368_1544 (size=12001) 2024-12-04T19:16:18,483 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/B/960c8e993e7b47bea1e536660db46177 2024-12-04T19:16:18,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/C/bd2202b4a0a54318a5b6cd829d6a1118 is 50, key is test_row_0/C:col10/1733339777255/Put/seqid=0 2024-12-04T19:16:18,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742369_1545 (size=12001) 2024-12-04T19:16:18,491 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/C/bd2202b4a0a54318a5b6cd829d6a1118 2024-12-04T19:16:18,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/A/6e8ada5fc5614ae6a4dea92a5a8ddd99 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/6e8ada5fc5614ae6a4dea92a5a8ddd99 2024-12-04T19:16:18,496 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/6e8ada5fc5614ae6a4dea92a5a8ddd99, entries=150, sequenceid=36, filesize=30.2 K 2024-12-04T19:16:18,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/B/960c8e993e7b47bea1e536660db46177 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/960c8e993e7b47bea1e536660db46177 2024-12-04T19:16:18,500 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/960c8e993e7b47bea1e536660db46177, entries=150, sequenceid=36, filesize=11.7 K 2024-12-04T19:16:18,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/C/bd2202b4a0a54318a5b6cd829d6a1118 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/bd2202b4a0a54318a5b6cd829d6a1118 2024-12-04T19:16:18,513 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/bd2202b4a0a54318a5b6cd829d6a1118, entries=150, sequenceid=36, filesize=11.7 K 2024-12-04T19:16:18,513 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for e22a2b2fd059f99265758fd39ea40832 in 495ms, sequenceid=36, compaction requested=false 2024-12-04T19:16:18,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2603): Flush status journal for e22a2b2fd059f99265758fd39ea40832: 2024-12-04T19:16:18,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. 2024-12-04T19:16:18,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=149 2024-12-04T19:16:18,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4169): Remote procedure done, pid=149 2024-12-04T19:16:18,516 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-12-04T19:16:18,516 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2650 sec 2024-12-04T19:16:18,519 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees in 1.2750 sec 2024-12-04T19:16:18,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(8855): Flush requested on e22a2b2fd059f99265758fd39ea40832 2024-12-04T19:16:18,524 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e22a2b2fd059f99265758fd39ea40832 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-04T19:16:18,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e22a2b2fd059f99265758fd39ea40832, store=A 2024-12-04T19:16:18,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:16:18,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e22a2b2fd059f99265758fd39ea40832, store=B 2024-12-04T19:16:18,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:16:18,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e22a2b2fd059f99265758fd39ea40832, store=C 2024-12-04T19:16:18,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:16:18,533 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412049d9f461835be48c393b66493cadaf452_e22a2b2fd059f99265758fd39ea40832 is 50, key is test_row_0/A:col10/1733339778523/Put/seqid=0 2024-12-04T19:16:18,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742370_1546 (size=12154) 2024-12-04T19:16:18,538 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:18,542 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412049d9f461835be48c393b66493cadaf452_e22a2b2fd059f99265758fd39ea40832 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412049d9f461835be48c393b66493cadaf452_e22a2b2fd059f99265758fd39ea40832 2024-12-04T19:16:18,543 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/A/cf2502f3a22c492db4dffecb5a394943, store: [table=TestAcidGuarantees family=A region=e22a2b2fd059f99265758fd39ea40832] 2024-12-04T19:16:18,543 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/A/cf2502f3a22c492db4dffecb5a394943 is 175, key is test_row_0/A:col10/1733339778523/Put/seqid=0 2024-12-04T19:16:18,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742371_1547 (size=30955) 2024-12-04T19:16:18,547 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:18,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38922 deadline: 1733339838543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:18,547 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:18,547 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:18,547 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:18,547 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:18,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38914 deadline: 1733339838544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:18,547 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:18,547 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:18,547 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:18,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:18,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38904 deadline: 1733339838547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:18,550 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:18,550 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:18,550 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:18,654 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:18,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38922 deadline: 1733339838652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:18,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:18,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38904 deadline: 1733339838652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:18,655 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:18,655 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:18,655 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:18,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:18,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38914 deadline: 1733339838652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:18,655 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:18,655 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:18,655 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:18,655 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:18,655 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:18,656 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:18,864 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:18,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38904 deadline: 1733339838861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:18,864 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:18,865 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:18,864 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:18,865 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:18,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38922 deadline: 1733339838861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:18,865 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:18,865 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:18,865 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:18,865 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:18,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38914 deadline: 1733339838861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:18,865 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:18,865 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:18,865 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:18,947 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=51, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/A/cf2502f3a22c492db4dffecb5a394943 2024-12-04T19:16:18,958 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/B/639b99abe7c64a479e7d2d343d01dc10 is 50, key is test_row_0/B:col10/1733339778523/Put/seqid=0 2024-12-04T19:16:18,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742372_1548 (size=12001) 2024-12-04T19:16:18,985 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/B/639b99abe7c64a479e7d2d343d01dc10 2024-12-04T19:16:19,002 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/C/6c24293234724f478ca10a56c33300d4 is 50, key is test_row_0/C:col10/1733339778523/Put/seqid=0 2024-12-04T19:16:19,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742373_1549 (size=12001) 2024-12-04T19:16:19,173 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:19,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38904 deadline: 1733339839172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:19,174 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:19,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38922 deadline: 1733339839172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:19,174 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:19,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:19,174 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:19,174 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:19,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38914 deadline: 1733339839172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:19,174 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:19,174 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:19,174 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:19,174 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:19,174 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:19,174 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:19,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-04T19:16:19,392 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T19:16:19,392 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T19:16:19,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees 2024-12-04T19:16:19,394 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T19:16:19,395 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T19:16:19,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-04T19:16:19,395 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T19:16:19,410 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/C/6c24293234724f478ca10a56c33300d4 2024-12-04T19:16:19,413 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/A/cf2502f3a22c492db4dffecb5a394943 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/cf2502f3a22c492db4dffecb5a394943 2024-12-04T19:16:19,416 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/cf2502f3a22c492db4dffecb5a394943, entries=150, sequenceid=51, filesize=30.2 K 2024-12-04T19:16:19,417 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/B/639b99abe7c64a479e7d2d343d01dc10 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/639b99abe7c64a479e7d2d343d01dc10 2024-12-04T19:16:19,419 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/639b99abe7c64a479e7d2d343d01dc10, entries=150, sequenceid=51, filesize=11.7 K 2024-12-04T19:16:19,419 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/C/6c24293234724f478ca10a56c33300d4 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/6c24293234724f478ca10a56c33300d4 2024-12-04T19:16:19,421 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/6c24293234724f478ca10a56c33300d4, entries=150, sequenceid=51, filesize=11.7 K 2024-12-04T19:16:19,422 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for e22a2b2fd059f99265758fd39ea40832 in 898ms, sequenceid=51, compaction requested=true 2024-12-04T19:16:19,422 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e22a2b2fd059f99265758fd39ea40832: 2024-12-04T19:16:19,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e22a2b2fd059f99265758fd39ea40832:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T19:16:19,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:16:19,422 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T19:16:19,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e22a2b2fd059f99265758fd39ea40832:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T19:16:19,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:16:19,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e22a2b2fd059f99265758fd39ea40832:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T19:16:19,422 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T19:16:19,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T19:16:19,425 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T19:16:19,425 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T19:16:19,425 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1541): e22a2b2fd059f99265758fd39ea40832/B is initiating minor compaction (all files) 2024-12-04T19:16:19,425 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1541): e22a2b2fd059f99265758fd39ea40832/A is initiating minor compaction (all files) 2024-12-04T19:16:19,425 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e22a2b2fd059f99265758fd39ea40832/B in TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. 2024-12-04T19:16:19,425 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e22a2b2fd059f99265758fd39ea40832/A in TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. 2024-12-04T19:16:19,425 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/fc097f27a101452eba3408fa6e892e3d, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/960c8e993e7b47bea1e536660db46177, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/639b99abe7c64a479e7d2d343d01dc10] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp, totalSize=35.2 K 2024-12-04T19:16:19,425 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/e9840326039a48a186a612cb083952c7, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/6e8ada5fc5614ae6a4dea92a5a8ddd99, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/cf2502f3a22c492db4dffecb5a394943] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp, totalSize=90.7 K 2024-12-04T19:16:19,425 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. 2024-12-04T19:16:19,425 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. files: [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/e9840326039a48a186a612cb083952c7, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/6e8ada5fc5614ae6a4dea92a5a8ddd99, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/cf2502f3a22c492db4dffecb5a394943] 2024-12-04T19:16:19,426 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting fc097f27a101452eba3408fa6e892e3d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1733339777221 2024-12-04T19:16:19,426 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting e9840326039a48a186a612cb083952c7, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1733339777221 2024-12-04T19:16:19,426 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 960c8e993e7b47bea1e536660db46177, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1733339777250 2024-12-04T19:16:19,426 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e8ada5fc5614ae6a4dea92a5a8ddd99, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1733339777250 2024-12-04T19:16:19,426 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 639b99abe7c64a479e7d2d343d01dc10, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733339778409 2024-12-04T19:16:19,426 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting cf2502f3a22c492db4dffecb5a394943, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733339778409 2024-12-04T19:16:19,432 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e22a2b2fd059f99265758fd39ea40832#B#compaction#467 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:16:19,433 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e22a2b2fd059f99265758fd39ea40832] 2024-12-04T19:16:19,433 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/B/6a48d09d5eab4be997bf96850b7c637e is 50, key is test_row_0/B:col10/1733339778523/Put/seqid=0 2024-12-04T19:16:19,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(8855): Flush requested on e22a2b2fd059f99265758fd39ea40832 2024-12-04T19:16:19,433 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e22a2b2fd059f99265758fd39ea40832 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-04T19:16:19,436 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120478517df512d244aca61e7377ac89c490_e22a2b2fd059f99265758fd39ea40832 store=[table=TestAcidGuarantees family=A region=e22a2b2fd059f99265758fd39ea40832] 2024-12-04T19:16:19,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742374_1550 (size=12104) 2024-12-04T19:16:19,437 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120478517df512d244aca61e7377ac89c490_e22a2b2fd059f99265758fd39ea40832, store=[table=TestAcidGuarantees family=A region=e22a2b2fd059f99265758fd39ea40832] 2024-12-04T19:16:19,437 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120478517df512d244aca61e7377ac89c490_e22a2b2fd059f99265758fd39ea40832 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e22a2b2fd059f99265758fd39ea40832] 2024-12-04T19:16:19,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742375_1551 (size=4469) 2024-12-04T19:16:19,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e22a2b2fd059f99265758fd39ea40832, store=A 2024-12-04T19:16:19,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:16:19,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e22a2b2fd059f99265758fd39ea40832, store=B 2024-12-04T19:16:19,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:16:19,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e22a2b2fd059f99265758fd39ea40832, store=C 2024-12-04T19:16:19,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:16:19,449 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120483703369dc0a4938a00cadc041f49dc8_e22a2b2fd059f99265758fd39ea40832 is 50, key is test_row_0/A:col10/1733339779432/Put/seqid=0 2024-12-04T19:16:19,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742376_1552 (size=14594) 2024-12-04T19:16:19,468 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:19,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38946 deadline: 1733339839464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:19,468 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:19,468 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:19,468 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:19,471 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:19,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38908 deadline: 1733339839468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:19,471 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:19,471 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:19,471 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:19,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-04T19:16:19,546 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-04T19:16:19,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. 2024-12-04T19:16:19,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. as already flushing 2024-12-04T19:16:19,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. 2024-12-04T19:16:19,547 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:19,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:19,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:19,573 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:19,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38946 deadline: 1733339839571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:19,573 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:19,573 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:19,573 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:19,583 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:19,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38908 deadline: 1733339839582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:19,583 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:19,583 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:19,583 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:19,683 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:19,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38904 deadline: 1733339839681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:19,683 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:19,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38922 deadline: 1733339839681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:19,683 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:19,684 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:19,684 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:19,684 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:19,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:19,684 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:19,684 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:19,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38914 deadline: 1733339839682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:19,684 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:19,684 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:19,684 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:19,699 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-04T19:16:19,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. 2024-12-04T19:16:19,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. as already flushing 2024-12-04T19:16:19,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. 2024-12-04T19:16:19,699 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:19,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:19,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:19,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-04T19:16:19,784 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:19,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38946 deadline: 1733339839782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:19,784 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:19,784 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:19,784 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:19,794 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:19,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38908 deadline: 1733339839792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:19,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:19,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:19,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:19,842 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/B/6a48d09d5eab4be997bf96850b7c637e as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/6a48d09d5eab4be997bf96850b7c637e 2024-12-04T19:16:19,842 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e22a2b2fd059f99265758fd39ea40832#A#compaction#468 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:16:19,843 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/A/f2964597d875459886ebf4e53112c0fa is 175, key is test_row_0/A:col10/1733339778523/Put/seqid=0 2024-12-04T19:16:19,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742377_1553 (size=31058) 2024-12-04T19:16:19,848 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e22a2b2fd059f99265758fd39ea40832/B of e22a2b2fd059f99265758fd39ea40832 into 6a48d09d5eab4be997bf96850b7c637e(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:16:19,848 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e22a2b2fd059f99265758fd39ea40832: 2024-12-04T19:16:19,848 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., storeName=e22a2b2fd059f99265758fd39ea40832/B, priority=13, startTime=1733339779422; duration=0sec 2024-12-04T19:16:19,848 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T19:16:19,848 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e22a2b2fd059f99265758fd39ea40832:B 2024-12-04T19:16:19,848 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T19:16:19,849 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T19:16:19,849 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1541): e22a2b2fd059f99265758fd39ea40832/C is initiating minor compaction (all files) 2024-12-04T19:16:19,849 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e22a2b2fd059f99265758fd39ea40832/C in TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. 2024-12-04T19:16:19,849 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/79ccd2ff28b54a96842ea98b8af18281, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/bd2202b4a0a54318a5b6cd829d6a1118, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/6c24293234724f478ca10a56c33300d4] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp, totalSize=35.2 K 2024-12-04T19:16:19,850 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 79ccd2ff28b54a96842ea98b8af18281, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1733339777221 2024-12-04T19:16:19,850 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting bd2202b4a0a54318a5b6cd829d6a1118, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1733339777250 2024-12-04T19:16:19,850 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c24293234724f478ca10a56c33300d4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733339778409 2024-12-04T19:16:19,852 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-04T19:16:19,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. 2024-12-04T19:16:19,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. as already flushing 2024-12-04T19:16:19,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. 2024-12-04T19:16:19,852 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:19,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:19,854 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:19,855 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e22a2b2fd059f99265758fd39ea40832#C#compaction#470 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:16:19,856 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/C/0127ace727de4eeca25fc4c7b6984a09 is 50, key is test_row_0/C:col10/1733339778523/Put/seqid=0 2024-12-04T19:16:19,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:19,859 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120483703369dc0a4938a00cadc041f49dc8_e22a2b2fd059f99265758fd39ea40832 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120483703369dc0a4938a00cadc041f49dc8_e22a2b2fd059f99265758fd39ea40832 2024-12-04T19:16:19,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742378_1554 (size=12104) 2024-12-04T19:16:19,860 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/A/a72a42035aef486594852110120ee0a2, store: [table=TestAcidGuarantees family=A region=e22a2b2fd059f99265758fd39ea40832] 2024-12-04T19:16:19,861 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/A/a72a42035aef486594852110120ee0a2 is 175, key is test_row_0/A:col10/1733339779432/Put/seqid=0 2024-12-04T19:16:19,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742379_1555 (size=39549) 2024-12-04T19:16:19,867 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=70, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/A/a72a42035aef486594852110120ee0a2 2024-12-04T19:16:19,873 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/C/0127ace727de4eeca25fc4c7b6984a09 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/0127ace727de4eeca25fc4c7b6984a09 2024-12-04T19:16:19,878 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e22a2b2fd059f99265758fd39ea40832/C of e22a2b2fd059f99265758fd39ea40832 into 0127ace727de4eeca25fc4c7b6984a09(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:16:19,878 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e22a2b2fd059f99265758fd39ea40832: 2024-12-04T19:16:19,878 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., storeName=e22a2b2fd059f99265758fd39ea40832/C, priority=13, startTime=1733339779422; duration=0sec 2024-12-04T19:16:19,878 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:16:19,878 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e22a2b2fd059f99265758fd39ea40832:C 2024-12-04T19:16:19,880 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/B/3997e2a8756e43b3ab90f0f40d46e8f4 is 50, key is test_row_0/B:col10/1733339779432/Put/seqid=0 2024-12-04T19:16:19,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742380_1556 (size=12001) 2024-12-04T19:16:19,898 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=70 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/B/3997e2a8756e43b3ab90f0f40d46e8f4 2024-12-04T19:16:19,905 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/C/01b887b3f0cd49d49bb361c4e67c6fc8 is 50, key is test_row_0/C:col10/1733339779432/Put/seqid=0 2024-12-04T19:16:19,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742381_1557 (size=12001) 2024-12-04T19:16:20,008 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-04T19:16:20,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. 2024-12-04T19:16:20,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. as already flushing 2024-12-04T19:16:20,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. 2024-12-04T19:16:20,008 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:20,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:20,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:20,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-04T19:16:20,103 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:20,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38946 deadline: 1733339840095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:20,103 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:20,103 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:20,103 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:20,110 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:20,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38908 deadline: 1733339840102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:20,111 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:20,111 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:20,111 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:20,164 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-04T19:16:20,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. 2024-12-04T19:16:20,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. as already flushing 2024-12-04T19:16:20,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. 2024-12-04T19:16:20,165 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:20,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:20,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:20,255 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/A/f2964597d875459886ebf4e53112c0fa as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/f2964597d875459886ebf4e53112c0fa 2024-12-04T19:16:20,259 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e22a2b2fd059f99265758fd39ea40832/A of e22a2b2fd059f99265758fd39ea40832 into f2964597d875459886ebf4e53112c0fa(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:16:20,259 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e22a2b2fd059f99265758fd39ea40832: 2024-12-04T19:16:20,259 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., storeName=e22a2b2fd059f99265758fd39ea40832/A, priority=13, startTime=1733339779422; duration=0sec 2024-12-04T19:16:20,260 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:16:20,260 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e22a2b2fd059f99265758fd39ea40832:A 2024-12-04T19:16:20,311 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=70 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/C/01b887b3f0cd49d49bb361c4e67c6fc8 2024-12-04T19:16:20,316 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/A/a72a42035aef486594852110120ee0a2 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/a72a42035aef486594852110120ee0a2 2024-12-04T19:16:20,317 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-04T19:16:20,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. 2024-12-04T19:16:20,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. as already flushing 2024-12-04T19:16:20,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. 2024-12-04T19:16:20,318 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:20,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:20,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:20,321 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/a72a42035aef486594852110120ee0a2, entries=200, sequenceid=70, filesize=38.6 K 2024-12-04T19:16:20,321 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/B/3997e2a8756e43b3ab90f0f40d46e8f4 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/3997e2a8756e43b3ab90f0f40d46e8f4 2024-12-04T19:16:20,325 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/3997e2a8756e43b3ab90f0f40d46e8f4, entries=150, sequenceid=70, filesize=11.7 K 2024-12-04T19:16:20,326 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/C/01b887b3f0cd49d49bb361c4e67c6fc8 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/01b887b3f0cd49d49bb361c4e67c6fc8 2024-12-04T19:16:20,330 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/01b887b3f0cd49d49bb361c4e67c6fc8, entries=150, sequenceid=70, filesize=11.7 K 2024-12-04T19:16:20,331 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for e22a2b2fd059f99265758fd39ea40832 in 898ms, sequenceid=70, compaction requested=false 2024-12-04T19:16:20,331 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e22a2b2fd059f99265758fd39ea40832: 2024-12-04T19:16:20,472 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-04T19:16:20,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. 2024-12-04T19:16:20,472 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2902): Flushing e22a2b2fd059f99265758fd39ea40832 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-04T19:16:20,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e22a2b2fd059f99265758fd39ea40832, store=A 2024-12-04T19:16:20,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:16:20,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e22a2b2fd059f99265758fd39ea40832, store=B 2024-12-04T19:16:20,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:16:20,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e22a2b2fd059f99265758fd39ea40832, store=C 2024-12-04T19:16:20,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:16:20,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412043256a685911b4829953bf407724e052e_e22a2b2fd059f99265758fd39ea40832 is 50, key is test_row_0/A:col10/1733339779467/Put/seqid=0 2024-12-04T19:16:20,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742382_1558 (size=12154) 2024-12-04T19:16:20,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-04T19:16:20,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(8855): Flush requested on e22a2b2fd059f99265758fd39ea40832 2024-12-04T19:16:20,612 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. as already flushing 2024-12-04T19:16:20,652 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:20,652 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:20,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38946 deadline: 1733339840650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:20,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38908 deadline: 1733339840650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:20,652 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:20,652 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:20,653 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:20,653 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:20,653 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:20,653 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:20,695 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:20,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38904 deadline: 1733339840692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:20,695 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:20,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38914 deadline: 1733339840692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:20,695 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 9ba2734cd39f,33969,1733339629791 for 'test_row_2' in e22a2b2fd059f99265758fd39ea40832 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2148 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-04T19:16:20,696 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:20,696 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:20,696 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:20,695 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 9ba2734cd39f,33969,1733339629791 for 'test_row_2' in e22a2b2fd059f99265758fd39ea40832 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2151 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-04T19:16:20,696 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:20,696 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:20,696 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:20,703 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:20,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38922 deadline: 1733339840701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:20,704 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 9ba2734cd39f,33969,1733339629791 for 'test_row_0' in e22a2b2fd059f99265758fd39ea40832 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-04T19:16:20,704 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:20,704 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:20,704 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:20,764 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:20,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38908 deadline: 1733339840762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:20,765 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:20,765 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:20,765 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:20,765 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:20,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38946 deadline: 1733339840762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:20,765 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:20,765 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:20,765 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:20,790 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-04T19:16:20,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:20,910 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412043256a685911b4829953bf407724e052e_e22a2b2fd059f99265758fd39ea40832 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412043256a685911b4829953bf407724e052e_e22a2b2fd059f99265758fd39ea40832 2024-12-04T19:16:20,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/A/7da7e18914a445289bf8345906b1e65f, store: [table=TestAcidGuarantees family=A region=e22a2b2fd059f99265758fd39ea40832] 2024-12-04T19:16:20,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/A/7da7e18914a445289bf8345906b1e65f is 175, key is test_row_0/A:col10/1733339779467/Put/seqid=0 2024-12-04T19:16:20,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742383_1559 (size=30955) 2024-12-04T19:16:20,973 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:20,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38908 deadline: 1733339840972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:20,974 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:20,974 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:20,974 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:20,974 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:20,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38946 deadline: 1733339840972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:20,974 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:20,974 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:20,974 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:21,283 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:21,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38908 deadline: 1733339841281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:21,284 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:21,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38946 deadline: 1733339841281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:21,284 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:21,284 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:21,284 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:21,284 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:21,284 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:21,284 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:21,315 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=86, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/A/7da7e18914a445289bf8345906b1e65f 2024-12-04T19:16:21,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/B/3c49a0fb6b1a42c1aa09c7126d22582c is 50, key is test_row_0/B:col10/1733339779467/Put/seqid=0 2024-12-04T19:16:21,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742384_1560 (size=12001) 2024-12-04T19:16:21,324 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/B/3c49a0fb6b1a42c1aa09c7126d22582c 2024-12-04T19:16:21,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/C/c06df5860725447fbac6bcef54a9402c is 50, key is test_row_0/C:col10/1733339779467/Put/seqid=0 2024-12-04T19:16:21,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742385_1561 (size=12001) 2024-12-04T19:16:21,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-04T19:16:21,732 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/C/c06df5860725447fbac6bcef54a9402c 2024-12-04T19:16:21,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/A/7da7e18914a445289bf8345906b1e65f as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/7da7e18914a445289bf8345906b1e65f 2024-12-04T19:16:21,738 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/7da7e18914a445289bf8345906b1e65f, entries=150, sequenceid=86, filesize=30.2 K 2024-12-04T19:16:21,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/B/3c49a0fb6b1a42c1aa09c7126d22582c as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/3c49a0fb6b1a42c1aa09c7126d22582c 2024-12-04T19:16:21,741 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/3c49a0fb6b1a42c1aa09c7126d22582c, entries=150, sequenceid=86, filesize=11.7 K 2024-12-04T19:16:21,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/C/c06df5860725447fbac6bcef54a9402c as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/c06df5860725447fbac6bcef54a9402c 2024-12-04T19:16:21,748 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/c06df5860725447fbac6bcef54a9402c, entries=150, sequenceid=86, filesize=11.7 K 2024-12-04T19:16:21,749 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for e22a2b2fd059f99265758fd39ea40832 in 1277ms, sequenceid=86, compaction requested=true 2024-12-04T19:16:21,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2603): Flush status journal for e22a2b2fd059f99265758fd39ea40832: 2024-12-04T19:16:21,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. 2024-12-04T19:16:21,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=151 2024-12-04T19:16:21,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4169): Remote procedure done, pid=151 2024-12-04T19:16:21,751 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-12-04T19:16:21,752 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3550 sec 2024-12-04T19:16:21,753 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees in 2.3600 sec 2024-12-04T19:16:21,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(8855): Flush requested on e22a2b2fd059f99265758fd39ea40832 2024-12-04T19:16:21,795 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e22a2b2fd059f99265758fd39ea40832 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-04T19:16:21,797 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733339780643, max timestamp=1733339781794Num uniques 859; 2024-12-04T19:16:21,797 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T19:16:21,797 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733339780643, max timestamp=1733339781794Num uniques 859; 2024-12-04T19:16:21,797 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733339780643, max timestamp=1733339781794Num uniques 859; 2024-12-04T19:16:21,797 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T19:16:21,797 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T19:16:21,798 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e22a2b2fd059f99265758fd39ea40832, store=A 2024-12-04T19:16:21,798 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T19:16:21,798 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e22a2b2fd059f99265758fd39ea40832, store=B 2024-12-04T19:16:21,798 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T19:16:21,798 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e22a2b2fd059f99265758fd39ea40832, store=C 2024-12-04T19:16:21,798 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T19:16:21,803 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412046dd229654b714e59bd1b3b3d132b21f4_e22a2b2fd059f99265758fd39ea40832 is 50, key is test_row_0/A:col10/1733339780649/Put/seqid=0 2024-12-04T19:16:21,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742386_1562 (size=14594) 2024-12-04T19:16:21,838 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733339781826, max timestamp=1733339781835Num uniques 859; 2024-12-04T19:16:21,838 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T19:16:21,840 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733339781826, max timestamp=1733339781835Num uniques 859; 2024-12-04T19:16:21,840 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-04T19:16:21,841 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733339781826, max timestamp=1733339781835Num uniques 859; 2024-12-04T19:16:21,841 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-04T19:16:21,884 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733339781865, max timestamp=1733339781880Num uniques 1018; 2024-12-04T19:16:21,884 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-04T19:16:21,887 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733339781865, max timestamp=1733339781880Num uniques 1009; 2024-12-04T19:16:21,887 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46213, new segment data size=6870, suffix heap size=132904,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=150 2024-12-04T19:16:21,890 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733339781865, max timestamp=1733339781880Num uniques 1009; 2024-12-04T19:16:21,890 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46213, new segment data size=6870, suffix heap size=132904,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=150 2024-12-04T19:16:21,940 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733339781909, max timestamp=1733339781936Num uniques 1009; 2024-12-04T19:16:21,940 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733339781909, max timestamp=1733339781936Num uniques 1027; 2024-12-04T19:16:21,940 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=8102, suffix heap size=132904,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=177 2024-12-04T19:16:21,940 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-12-04T19:16:21,940 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733339781909, max timestamp=1733339781936Num uniques 1009; 2024-12-04T19:16:21,940 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=8102, suffix heap size=132904,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=177 2024-12-04T19:16:21,998 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1733339781981, max timestamp=1733339781995Num uniques 1036; 2024-12-04T19:16:21,998 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=10806, suffix heap size=135712,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=236 2024-12-04T19:16:21,998 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1733339781981, max timestamp=1733339781995Num uniques 1036; 2024-12-04T19:16:21,998 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=10806, suffix heap size=135712,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=236 2024-12-04T19:16:21,998 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1733339781981, max timestamp=1733339781995Num uniques 1036; 2024-12-04T19:16:21,998 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=10806, suffix heap size=135712,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=236 2024-12-04T19:16:22,038 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1733339782026, max timestamp=1733339782035Num uniques 1095; 2024-12-04T19:16:22,038 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=11220, suffix heap size=141848,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=245 2024-12-04T19:16:22,039 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1733339782026, max timestamp=1733339782035Num uniques 1095; 2024-12-04T19:16:22,039 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1733339782026, max timestamp=1733339782035Num uniques 1095; 2024-12-04T19:16:22,039 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=11220, suffix heap size=141848,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=245 2024-12-04T19:16:22,039 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=11220, suffix heap size=141848,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=245 2024-12-04T19:16:22,088 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1733339782054, max timestamp=1733339782085Num uniques 1104; 2024-12-04T19:16:22,088 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=7050, suffix heap size=142784,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=154 2024-12-04T19:16:22,088 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1733339782054, max timestamp=1733339782085Num uniques 1104; 2024-12-04T19:16:22,088 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=7050, suffix heap size=142784,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=154 2024-12-04T19:16:22,088 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1733339782054, max timestamp=1733339782085Num uniques 1104; 2024-12-04T19:16:22,088 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=7050, suffix heap size=142784,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=154 2024-12-04T19:16:22,133 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1733339782096, max timestamp=1733339782130Num uniques 1013; 2024-12-04T19:16:22,133 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1733339782096, max timestamp=1733339782130Num uniques 1013; 2024-12-04T19:16:22,133 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1733339782096, max timestamp=1733339782130Num uniques 1013; 2024-12-04T19:16:22,133 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=9748, suffix heap size=133320,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=213 2024-12-04T19:16:22,133 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=9748, suffix heap size=133320,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=213 2024-12-04T19:16:22,133 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=9748, suffix heap size=133320,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=213 2024-12-04T19:16:22,192 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1733339782130, max timestamp=1733339782189Num uniques 1072; 2024-12-04T19:16:22,192 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=10162, suffix heap size=139456,new segment heap size=23328 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=222 2024-12-04T19:16:22,192 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1733339782130, max timestamp=1733339782189Num uniques 1072; 2024-12-04T19:16:22,192 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=10162, suffix heap size=139456,new segment heap size=23328 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=222 2024-12-04T19:16:22,192 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1733339782130, max timestamp=1733339782189Num uniques 1072; 2024-12-04T19:16:22,192 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=10162, suffix heap size=139456,new segment heap size=23328 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=222 2024-12-04T19:16:22,207 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:22,210 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412046dd229654b714e59bd1b3b3d132b21f4_e22a2b2fd059f99265758fd39ea40832 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412046dd229654b714e59bd1b3b3d132b21f4_e22a2b2fd059f99265758fd39ea40832 2024-12-04T19:16:22,211 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/A/57159d1437b94fd2bcfa60f09c0102c2, store: [table=TestAcidGuarantees family=A region=e22a2b2fd059f99265758fd39ea40832] 2024-12-04T19:16:22,211 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/A/57159d1437b94fd2bcfa60f09c0102c2 is 175, key is test_row_0/A:col10/1733339780649/Put/seqid=0 2024-12-04T19:16:22,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742387_1563 (size=39549) 2024-12-04T19:16:22,239 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1733339782208, max timestamp=1733339782236Num uniques 1081; 2024-12-04T19:16:22,239 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49506, new segment data size=8286, suffix heap size=140392,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1081, new segment cells count=181 2024-12-04T19:16:22,239 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8276, totalHeapSize=19064, min timestamp=1733339782208, max timestamp=1733339782237Num uniques 1081; 2024-12-04T19:16:22,239 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49496, new segment data size=8276, suffix heap size=140392,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1081, new segment cells count=181 2024-12-04T19:16:22,240 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8283, totalHeapSize=19064, min timestamp=1733339782208, max timestamp=1733339782237Num uniques 1081; 2024-12-04T19:16:22,240 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49503, new segment data size=8283, suffix heap size=140392,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1081, new segment cells count=181 2024-12-04T19:16:22,283 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=190, cellSize=8700, totalHeapSize=20000, min timestamp=1733339782270, max timestamp=1733339782281Num uniques 1040; 2024-12-04T19:16:22,283 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=8700, suffix heap size=136128,new segment heap size=20000 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=190 2024-12-04T19:16:22,284 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=190, cellSize=8700, totalHeapSize=20000, min timestamp=1733339782270, max timestamp=1733339782281Num uniques 1040; 2024-12-04T19:16:22,284 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=8700, suffix heap size=136128,new segment heap size=20000 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=190 2024-12-04T19:16:22,284 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=190, cellSize=8700, totalHeapSize=20000, min timestamp=1733339782270, max timestamp=1733339782281Num uniques 1040; 2024-12-04T19:16:22,284 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=8700, suffix heap size=136128,new segment heap size=20000 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=190 2024-12-04T19:16:22,325 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=199, cellSize=9114, totalHeapSize=20936, min timestamp=1733339782312, max timestamp=1733339782322Num uniques 1049; 2024-12-04T19:16:22,325 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48044, new segment data size=9114, suffix heap size=137064,new segment heap size=20936 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1049, new segment cells count=199 2024-12-04T19:16:22,325 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=199, cellSize=9114, totalHeapSize=20936, min timestamp=1733339782312, max timestamp=1733339782322Num uniques 1049; 2024-12-04T19:16:22,325 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48044, new segment data size=9114, suffix heap size=137064,new segment heap size=20936 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1049, new segment cells count=199 2024-12-04T19:16:22,327 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=199, cellSize=9114, totalHeapSize=20936, min timestamp=1733339782312, max timestamp=1733339782322Num uniques 1049; 2024-12-04T19:16:22,327 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48044, new segment data size=9114, suffix heap size=137064,new segment heap size=20936 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1049, new segment cells count=199 2024-12-04T19:16:22,367 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=158, cellSize=7230, totalHeapSize=16672, min timestamp=1733339782356, max timestamp=1733339782364Num uniques 1058; 2024-12-04T19:16:22,368 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48450, new segment data size=7230, suffix heap size=138000,new segment heap size=16672 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1058, new segment cells count=158 2024-12-04T19:16:22,368 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=158, cellSize=7230, totalHeapSize=16672, min timestamp=1733339782356, max timestamp=1733339782364Num uniques 1058; 2024-12-04T19:16:22,368 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48450, new segment data size=7230, suffix heap size=138000,new segment heap size=16672 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1058, new segment cells count=158 2024-12-04T19:16:22,370 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=158, cellSize=7230, totalHeapSize=16672, min timestamp=1733339782356, max timestamp=1733339782364Num uniques 1058; 2024-12-04T19:16:22,370 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48450, new segment data size=7230, suffix heap size=138000,new segment heap size=16672 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1058, new segment cells count=158 2024-12-04T19:16:22,401 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=217, cellSize=9932, totalHeapSize=22808, min timestamp=1733339782390, max timestamp=1733339782398Num uniques 1017; 2024-12-04T19:16:22,401 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=217, cellSize=9932, totalHeapSize=22808, min timestamp=1733339782390, max timestamp=1733339782398Num uniques 1017; 2024-12-04T19:16:22,401 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46572, new segment data size=9932, suffix heap size=133736,new segment heap size=22808 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1017, new segment cells count=217 2024-12-04T19:16:22,401 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46572, new segment data size=9932, suffix heap size=133736,new segment heap size=22808 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1017, new segment cells count=217 2024-12-04T19:16:22,401 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=217, cellSize=9932, totalHeapSize=22808, min timestamp=1733339782390, max timestamp=1733339782398Num uniques 1017; 2024-12-04T19:16:22,401 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46572, new segment data size=9932, suffix heap size=133736,new segment heap size=22808 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1017, new segment cells count=217 2024-12-04T19:16:22,434 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=176, cellSize=8056, totalHeapSize=18544, min timestamp=1733339782420, max timestamp=1733339782431Num uniques 1076; 2024-12-04T19:16:22,434 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49276, new segment data size=8056, suffix heap size=139872,new segment heap size=18544 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1076, new segment cells count=176 2024-12-04T19:16:22,434 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=176, cellSize=8056, totalHeapSize=18544, min timestamp=1733339782420, max timestamp=1733339782431Num uniques 1076; 2024-12-04T19:16:22,434 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49276, new segment data size=8056, suffix heap size=139872,new segment heap size=18544 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1076, new segment cells count=176 2024-12-04T19:16:22,436 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=176, cellSize=8056, totalHeapSize=18544, min timestamp=1733339782420, max timestamp=1733339782431Num uniques 1076; 2024-12-04T19:16:22,436 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49276, new segment data size=8056, suffix heap size=139872,new segment heap size=18544 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1076, new segment cells count=176 2024-12-04T19:16:22,476 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=235, cellSize=10760, totalHeapSize=24680, min timestamp=1733339782457, max timestamp=1733339782473Num uniques 1035; 2024-12-04T19:16:22,476 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=235, cellSize=10760, totalHeapSize=24680, min timestamp=1733339782457, max timestamp=1733339782473Num uniques 1035; 2024-12-04T19:16:22,476 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47400, new segment data size=10760, suffix heap size=135608,new segment heap size=24680 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1035, new segment cells count=235 2024-12-04T19:16:22,476 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47400, new segment data size=10760, suffix heap size=135608,new segment heap size=24680 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1035, new segment cells count=235 2024-12-04T19:16:22,476 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=235, cellSize=10760, totalHeapSize=24680, min timestamp=1733339782457, max timestamp=1733339782473Num uniques 1035; 2024-12-04T19:16:22,476 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47400, new segment data size=10760, suffix heap size=135608,new segment heap size=24680 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1035, new segment cells count=235 2024-12-04T19:16:22,517 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=244, cellSize=11174, totalHeapSize=25616, min timestamp=1733339782489, max timestamp=1733339782514Num uniques 1094; 2024-12-04T19:16:22,517 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=50104, new segment data size=11174, suffix heap size=141744,new segment heap size=25616 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1094, new segment cells count=244 2024-12-04T19:16:22,517 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=244, cellSize=11174, totalHeapSize=25616, min timestamp=1733339782489, max timestamp=1733339782514Num uniques 1094; 2024-12-04T19:16:22,517 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=50104, new segment data size=11174, suffix heap size=141744,new segment heap size=25616 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1094, new segment cells count=244 2024-12-04T19:16:22,518 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=244, cellSize=11174, totalHeapSize=25616, min timestamp=1733339782489, max timestamp=1733339782514Num uniques 1094; 2024-12-04T19:16:22,518 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=50104, new segment data size=11174, suffix heap size=141744,new segment heap size=25616 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1094, new segment cells count=244 2024-12-04T19:16:22,556 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:22,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38946 deadline: 1733339842553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:22,556 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=203, cellSize=9295, totalHeapSize=21352, min timestamp=1733339782543, max timestamp=1733339782553Num uniques 1103; 2024-12-04T19:16:22,556 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=50515, new segment data size=9295, suffix heap size=142680,new segment heap size=21352 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1103, new segment cells count=203 2024-12-04T19:16:22,556 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=203, cellSize=9295, totalHeapSize=21352, min timestamp=1733339782543, max timestamp=1733339782553Num uniques 1103; 2024-12-04T19:16:22,556 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=203, cellSize=9295, totalHeapSize=21352, min timestamp=1733339782543, max timestamp=1733339782553Num uniques 1103; 2024-12-04T19:16:22,556 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=50515, new segment data size=9295, suffix heap size=142680,new segment heap size=21352 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1103, new segment cells count=203 2024-12-04T19:16:22,556 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=50515, new segment data size=9295, suffix heap size=142680,new segment heap size=21352 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1103, new segment cells count=203 2024-12-04T19:16:22,556 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:22,556 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:22,557 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:22,615 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=108, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/A/57159d1437b94fd2bcfa60f09c0102c2 2024-12-04T19:16:22,621 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/B/64018f244a93466aa3eb37e222b79cda is 50, key is test_row_0/B:col10/1733339780649/Put/seqid=0 2024-12-04T19:16:22,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742388_1564 (size=12001) 2024-12-04T19:16:22,628 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=108 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/B/64018f244a93466aa3eb37e222b79cda 2024-12-04T19:16:22,635 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=162, cellSize=7412, totalHeapSize=17088, min timestamp=1733339782601, max timestamp=1733339782624Num uniques 1062; 2024-12-04T19:16:22,635 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48632, new segment data size=7412, suffix heap size=138416,new segment heap size=17088 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1062, new segment cells count=162 2024-12-04T19:16:22,636 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=162, cellSize=7412, totalHeapSize=17088, min timestamp=1733339782601, max timestamp=1733339782624Num uniques 1062; 2024-12-04T19:16:22,636 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48632, new segment data size=7412, suffix heap size=138416,new segment heap size=17088 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1062, new segment cells count=162 2024-12-04T19:16:22,636 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=162, cellSize=7412, totalHeapSize=17088, min timestamp=1733339782601, max timestamp=1733339782624Num uniques 1062; 2024-12-04T19:16:22,636 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48632, new segment data size=7412, suffix heap size=138416,new segment heap size=17088 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1062, new segment cells count=162 2024-12-04T19:16:22,643 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/C/e284e87bc2f643d9a1018db9a18f4560 is 50, key is test_row_0/C:col10/1733339780649/Put/seqid=0 2024-12-04T19:16:22,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742389_1565 (size=12001) 2024-12-04T19:16:22,666 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=108 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/C/e284e87bc2f643d9a1018db9a18f4560 2024-12-04T19:16:22,670 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/A/57159d1437b94fd2bcfa60f09c0102c2 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/57159d1437b94fd2bcfa60f09c0102c2 2024-12-04T19:16:22,674 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/57159d1437b94fd2bcfa60f09c0102c2, entries=200, sequenceid=108, filesize=38.6 K 2024-12-04T19:16:22,674 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/B/64018f244a93466aa3eb37e222b79cda as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/64018f244a93466aa3eb37e222b79cda 2024-12-04T19:16:22,678 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/64018f244a93466aa3eb37e222b79cda, entries=150, sequenceid=108, filesize=11.7 K 2024-12-04T19:16:22,678 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/C/e284e87bc2f643d9a1018db9a18f4560 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/e284e87bc2f643d9a1018db9a18f4560 2024-12-04T19:16:22,682 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/e284e87bc2f643d9a1018db9a18f4560, entries=150, sequenceid=108, filesize=11.7 K 2024-12-04T19:16:22,683 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~86.11 KB/88176, currentSize=114.05 KB/116790 for e22a2b2fd059f99265758fd39ea40832 in 888ms, sequenceid=108, compaction requested=true 2024-12-04T19:16:22,683 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e22a2b2fd059f99265758fd39ea40832: 2024-12-04T19:16:22,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e22a2b2fd059f99265758fd39ea40832:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T19:16:22,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:16:22,683 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T19:16:22,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e22a2b2fd059f99265758fd39ea40832:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T19:16:22,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:16:22,683 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T19:16:22,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e22a2b2fd059f99265758fd39ea40832:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T19:16:22,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T19:16:22,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(8855): Flush requested on e22a2b2fd059f99265758fd39ea40832 2024-12-04T19:16:22,684 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e22a2b2fd059f99265758fd39ea40832 3/3 column families, dataSize=120.76 KB heapSize=344.86 KB 2024-12-04T19:16:22,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e22a2b2fd059f99265758fd39ea40832, store=A 2024-12-04T19:16:22,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T19:16:22,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e22a2b2fd059f99265758fd39ea40832, store=B 2024-12-04T19:16:22,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T19:16:22,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e22a2b2fd059f99265758fd39ea40832, store=C 2024-12-04T19:16:22,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T19:16:22,684 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141111 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T19:16:22,684 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T19:16:22,685 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1541): e22a2b2fd059f99265758fd39ea40832/A is initiating minor compaction (all files) 2024-12-04T19:16:22,685 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1541): e22a2b2fd059f99265758fd39ea40832/B is initiating minor compaction (all files) 2024-12-04T19:16:22,685 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e22a2b2fd059f99265758fd39ea40832/B in TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. 2024-12-04T19:16:22,685 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e22a2b2fd059f99265758fd39ea40832/A in TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. 2024-12-04T19:16:22,685 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/6a48d09d5eab4be997bf96850b7c637e, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/3997e2a8756e43b3ab90f0f40d46e8f4, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/3c49a0fb6b1a42c1aa09c7126d22582c, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/64018f244a93466aa3eb37e222b79cda] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp, totalSize=47.0 K 2024-12-04T19:16:22,685 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/f2964597d875459886ebf4e53112c0fa, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/a72a42035aef486594852110120ee0a2, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/7da7e18914a445289bf8345906b1e65f, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/57159d1437b94fd2bcfa60f09c0102c2] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp, totalSize=137.8 K 2024-12-04T19:16:22,685 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. 2024-12-04T19:16:22,685 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. files: [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/f2964597d875459886ebf4e53112c0fa, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/a72a42035aef486594852110120ee0a2, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/7da7e18914a445289bf8345906b1e65f, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/57159d1437b94fd2bcfa60f09c0102c2] 2024-12-04T19:16:22,685 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting f2964597d875459886ebf4e53112c0fa, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733339778409 2024-12-04T19:16:22,685 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a48d09d5eab4be997bf96850b7c637e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733339778409 2024-12-04T19:16:22,685 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting a72a42035aef486594852110120ee0a2, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=70, earliestPutTs=1733339778537 2024-12-04T19:16:22,685 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 3997e2a8756e43b3ab90f0f40d46e8f4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=70, earliestPutTs=1733339778546 2024-12-04T19:16:22,686 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7da7e18914a445289bf8345906b1e65f, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1733339779462 2024-12-04T19:16:22,686 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c49a0fb6b1a42c1aa09c7126d22582c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1733339779462 2024-12-04T19:16:22,686 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 57159d1437b94fd2bcfa60f09c0102c2, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1733339780643 2024-12-04T19:16:22,686 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 64018f244a93466aa3eb37e222b79cda, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1733339780643 2024-12-04T19:16:22,692 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412048fae1f7f6c09490ba3eb1f4273545f21_e22a2b2fd059f99265758fd39ea40832 is 50, key is test_row_0/A:col10/1733339782676/Put/seqid=0 2024-12-04T19:16:22,694 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e22a2b2fd059f99265758fd39ea40832] 2024-12-04T19:16:22,695 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e22a2b2fd059f99265758fd39ea40832#B#compaction#480 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:16:22,695 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/B/99b16982d0a4470bb478a045ef63b33f is 50, key is test_row_0/B:col10/1733339780649/Put/seqid=0 2024-12-04T19:16:22,697 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412047cb79e7174d44fb69f1006424e89ef93_e22a2b2fd059f99265758fd39ea40832 store=[table=TestAcidGuarantees family=A region=e22a2b2fd059f99265758fd39ea40832] 2024-12-04T19:16:22,699 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412047cb79e7174d44fb69f1006424e89ef93_e22a2b2fd059f99265758fd39ea40832, store=[table=TestAcidGuarantees family=A region=e22a2b2fd059f99265758fd39ea40832] 2024-12-04T19:16:22,699 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412047cb79e7174d44fb69f1006424e89ef93_e22a2b2fd059f99265758fd39ea40832 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e22a2b2fd059f99265758fd39ea40832] 2024-12-04T19:16:22,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742390_1566 (size=17534) 2024-12-04T19:16:22,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742391_1567 (size=12241) 2024-12-04T19:16:22,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742392_1568 (size=4469) 2024-12-04T19:16:22,713 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/B/99b16982d0a4470bb478a045ef63b33f as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/99b16982d0a4470bb478a045ef63b33f 2024-12-04T19:16:22,717 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e22a2b2fd059f99265758fd39ea40832/B of e22a2b2fd059f99265758fd39ea40832 into 99b16982d0a4470bb478a045ef63b33f(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:16:22,717 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e22a2b2fd059f99265758fd39ea40832: 2024-12-04T19:16:22,717 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., storeName=e22a2b2fd059f99265758fd39ea40832/B, priority=12, startTime=1733339782683; duration=0sec 2024-12-04T19:16:22,717 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T19:16:22,717 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e22a2b2fd059f99265758fd39ea40832:B 2024-12-04T19:16:22,717 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T19:16:22,717 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:22,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38908 deadline: 1733339842713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:22,718 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T19:16:22,718 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1541): e22a2b2fd059f99265758fd39ea40832/C is initiating minor compaction (all files) 2024-12-04T19:16:22,718 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:22,718 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e22a2b2fd059f99265758fd39ea40832/C in TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. 2024-12-04T19:16:22,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38946 deadline: 1733339842714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:22,718 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/0127ace727de4eeca25fc4c7b6984a09, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/01b887b3f0cd49d49bb361c4e67c6fc8, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/c06df5860725447fbac6bcef54a9402c, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/e284e87bc2f643d9a1018db9a18f4560] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp, totalSize=47.0 K 2024-12-04T19:16:22,718 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:22,718 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:22,718 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:22,718 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:22,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38914 deadline: 1733339842714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:22,718 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:22,718 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:22,718 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:22,718 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:22,718 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:22,718 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:22,719 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 0127ace727de4eeca25fc4c7b6984a09, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733339778409 2024-12-04T19:16:22,719 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 01b887b3f0cd49d49bb361c4e67c6fc8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=70, earliestPutTs=1733339778546 2024-12-04T19:16:22,719 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting c06df5860725447fbac6bcef54a9402c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1733339779462 2024-12-04T19:16:22,719 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting e284e87bc2f643d9a1018db9a18f4560, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1733339780643 2024-12-04T19:16:22,724 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:22,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38904 deadline: 1733339842724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:22,726 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 9ba2734cd39f,33969,1733339629791 for 'test_row_2' in e22a2b2fd059f99265758fd39ea40832 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-04T19:16:22,727 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:22,727 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:22,727 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:22,731 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:22,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38922 deadline: 1733339842728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:22,732 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e22a2b2fd059f99265758fd39ea40832#C#compaction#482 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:16:22,731 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 9ba2734cd39f,33969,1733339629791 for 'test_row_0' in e22a2b2fd059f99265758fd39ea40832 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-04T19:16:22,732 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:22,732 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/C/7c1857b1f5d544d0a6a56699ea2b9e6a is 50, key is test_row_0/C:col10/1733339780649/Put/seqid=0 2024-12-04T19:16:22,732 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:22,732 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:22,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742393_1569 (size=12241) 2024-12-04T19:16:22,825 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:22,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38946 deadline: 1733339842822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:22,826 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:22,826 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:22,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38914 deadline: 1733339842822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:22,826 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:22,826 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:22,826 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:22,826 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:22,826 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:22,826 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:22,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38908 deadline: 1733339842822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:22,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:22,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:22,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:23,033 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:23,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38946 deadline: 1733339843031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:23,034 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:23,034 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:23,034 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:23,034 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:23,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38908 deadline: 1733339843031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:23,034 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:23,034 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:23,034 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:23,035 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:23,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38914 deadline: 1733339843032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:23,035 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:23,035 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:23,035 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:23,105 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:23,108 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412048fae1f7f6c09490ba3eb1f4273545f21_e22a2b2fd059f99265758fd39ea40832 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412048fae1f7f6c09490ba3eb1f4273545f21_e22a2b2fd059f99265758fd39ea40832 2024-12-04T19:16:23,108 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/A/9f48a10803ae45208609adcf876e1759, store: [table=TestAcidGuarantees family=A region=e22a2b2fd059f99265758fd39ea40832] 2024-12-04T19:16:23,109 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/A/9f48a10803ae45208609adcf876e1759 is 175, key is test_row_0/A:col10/1733339782676/Put/seqid=0 2024-12-04T19:16:23,111 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e22a2b2fd059f99265758fd39ea40832#A#compaction#481 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:16:23,112 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/A/fb29d6266a644dec9c81e077a70bccec is 175, key is test_row_0/A:col10/1733339780649/Put/seqid=0 2024-12-04T19:16:23,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742394_1570 (size=48639) 2024-12-04T19:16:23,117 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=435, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/A/9f48a10803ae45208609adcf876e1759 2024-12-04T19:16:23,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742395_1571 (size=31195) 2024-12-04T19:16:23,124 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/A/fb29d6266a644dec9c81e077a70bccec as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/fb29d6266a644dec9c81e077a70bccec 2024-12-04T19:16:23,125 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/B/785e4c4c9f7e4d91975486886083afae is 50, key is test_row_0/B:col10/1733339782676/Put/seqid=0 2024-12-04T19:16:23,134 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e22a2b2fd059f99265758fd39ea40832/A of e22a2b2fd059f99265758fd39ea40832 into fb29d6266a644dec9c81e077a70bccec(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:16:23,134 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e22a2b2fd059f99265758fd39ea40832: 2024-12-04T19:16:23,134 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., storeName=e22a2b2fd059f99265758fd39ea40832/A, priority=12, startTime=1733339782683; duration=0sec 2024-12-04T19:16:23,134 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:16:23,134 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e22a2b2fd059f99265758fd39ea40832:A 2024-12-04T19:16:23,144 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/C/7c1857b1f5d544d0a6a56699ea2b9e6a as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/7c1857b1f5d544d0a6a56699ea2b9e6a 2024-12-04T19:16:23,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742396_1572 (size=12301) 2024-12-04T19:16:23,155 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e22a2b2fd059f99265758fd39ea40832/C of e22a2b2fd059f99265758fd39ea40832 into 7c1857b1f5d544d0a6a56699ea2b9e6a(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:16:23,155 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e22a2b2fd059f99265758fd39ea40832: 2024-12-04T19:16:23,155 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., storeName=e22a2b2fd059f99265758fd39ea40832/C, priority=12, startTime=1733339782683; duration=0sec 2024-12-04T19:16:23,155 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=435 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/B/785e4c4c9f7e4d91975486886083afae 2024-12-04T19:16:23,155 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:16:23,155 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e22a2b2fd059f99265758fd39ea40832:C 2024-12-04T19:16:23,171 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/C/5aaea413efe6478e9485ac530987a91f is 50, key is test_row_0/C:col10/1733339782676/Put/seqid=0 2024-12-04T19:16:23,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742397_1573 (size=12301) 2024-12-04T19:16:23,344 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:23,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38946 deadline: 1733339843342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:23,344 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:23,344 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:23,344 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:23,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38908 deadline: 1733339843342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:23,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38914 deadline: 1733339843342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:23,344 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:23,344 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:23,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:23,345 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:23,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:23,345 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:23,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:23,345 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:23,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-04T19:16:23,571 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T19:16:23,572 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T19:16:23,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-12-04T19:16:23,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-04T19:16:23,573 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T19:16:23,574 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T19:16:23,574 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T19:16:23,574 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=435 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/C/5aaea413efe6478e9485ac530987a91f 2024-12-04T19:16:23,578 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/A/9f48a10803ae45208609adcf876e1759 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/9f48a10803ae45208609adcf876e1759 2024-12-04T19:16:23,580 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/9f48a10803ae45208609adcf876e1759, entries=250, sequenceid=435, filesize=47.5 K 2024-12-04T19:16:23,581 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/B/785e4c4c9f7e4d91975486886083afae as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/785e4c4c9f7e4d91975486886083afae 2024-12-04T19:16:23,584 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/785e4c4c9f7e4d91975486886083afae, entries=150, sequenceid=435, filesize=12.0 K 2024-12-04T19:16:23,584 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/C/5aaea413efe6478e9485ac530987a91f as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/5aaea413efe6478e9485ac530987a91f 2024-12-04T19:16:23,587 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/5aaea413efe6478e9485ac530987a91f, entries=150, sequenceid=435, filesize=12.0 K 2024-12-04T19:16:23,588 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~344.81 KB/353088, currentSize=60.38 KB/61830 for e22a2b2fd059f99265758fd39ea40832 in 903ms, sequenceid=435, compaction requested=false 2024-12-04T19:16:23,588 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e22a2b2fd059f99265758fd39ea40832: 2024-12-04T19:16:23,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-04T19:16:23,727 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-04T19:16:23,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. 2024-12-04T19:16:23,727 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing e22a2b2fd059f99265758fd39ea40832 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-04T19:16:23,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e22a2b2fd059f99265758fd39ea40832, store=A 2024-12-04T19:16:23,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:16:23,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e22a2b2fd059f99265758fd39ea40832, store=B 2024-12-04T19:16:23,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:16:23,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e22a2b2fd059f99265758fd39ea40832, store=C 2024-12-04T19:16:23,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:16:23,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412049e8c2dfeab6c480a8f0fbe9cc2748dc4_e22a2b2fd059f99265758fd39ea40832 is 50, key is test_row_0/A:col10/1733339782712/Put/seqid=0 2024-12-04T19:16:23,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742398_1574 (size=12454) 2024-12-04T19:16:23,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(8855): Flush requested on e22a2b2fd059f99265758fd39ea40832 2024-12-04T19:16:23,854 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. as already flushing 2024-12-04T19:16:23,876 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:23,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 222 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38908 deadline: 1733339843873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:23,876 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:23,876 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:23,876 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:23,877 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733339783870, max timestamp=1733339783875Num uniques 859; 2024-12-04T19:16:23,877 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1733339783870, max timestamp=1733339783875Num uniques 859; 2024-12-04T19:16:23,877 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T19:16:23,877 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T19:16:23,877 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1733339783870, max timestamp=1733339783875Num uniques 859; 2024-12-04T19:16:23,877 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T19:16:23,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-04T19:16:23,904 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:23,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38914 deadline: 1733339843901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:23,905 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:23,905 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:23,905 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:23,907 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:23,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38946 deadline: 1733339843905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:23,908 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:23,908 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:23,908 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:23,985 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:23,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 223 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38908 deadline: 1733339843982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:23,985 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:23,985 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:23,985 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:24,014 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:24,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38914 deadline: 1733339844012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:24,014 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:24,014 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:24,014 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:24,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38946 deadline: 1733339844012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:24,014 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:24,015 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:24,015 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:24,015 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:24,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:24,138 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412049e8c2dfeab6c480a8f0fbe9cc2748dc4_e22a2b2fd059f99265758fd39ea40832 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412049e8c2dfeab6c480a8f0fbe9cc2748dc4_e22a2b2fd059f99265758fd39ea40832 2024-12-04T19:16:24,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/A/a2b6fc038d424b628b70340cbe9f34ed, store: [table=TestAcidGuarantees family=A region=e22a2b2fd059f99265758fd39ea40832] 2024-12-04T19:16:24,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/A/a2b6fc038d424b628b70340cbe9f34ed is 175, key is test_row_0/A:col10/1733339782712/Put/seqid=0 2024-12-04T19:16:24,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742399_1575 (size=31255) 2024-12-04T19:16:24,146 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=450, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/A/a2b6fc038d424b628b70340cbe9f34ed 2024-12-04T19:16:24,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/B/ecdd77d1d2374e438bd9b2dd6b202a8d is 50, key is test_row_0/B:col10/1733339782712/Put/seqid=0 2024-12-04T19:16:24,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742400_1576 (size=12301) 2024-12-04T19:16:24,158 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=450 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/B/ecdd77d1d2374e438bd9b2dd6b202a8d 2024-12-04T19:16:24,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/C/becd0bfa340543349ddc8f2e15c8f77d is 50, key is test_row_0/C:col10/1733339782712/Put/seqid=0 2024-12-04T19:16:24,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742401_1577 (size=12301) 2024-12-04T19:16:24,194 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:24,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 224 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38908 deadline: 1733339844191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:24,194 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:24,194 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:24,194 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:24,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-04T19:16:24,225 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:24,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38914 deadline: 1733339844222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:24,226 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:24,226 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:24,226 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:24,226 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:24,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38946 deadline: 1733339844222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:24,226 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:24,226 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:24,226 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:24,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:24,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 225 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38908 deadline: 1733339844502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:24,503 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:24,504 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:24,504 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:24,533 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:24,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38914 deadline: 1733339844531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:24,533 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:24,533 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:24,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38946 deadline: 1733339844531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:24,533 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:24,533 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:24,533 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:24,533 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:24,533 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:24,568 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=450 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/C/becd0bfa340543349ddc8f2e15c8f77d 2024-12-04T19:16:24,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/A/a2b6fc038d424b628b70340cbe9f34ed as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/a2b6fc038d424b628b70340cbe9f34ed 2024-12-04T19:16:24,573 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/a2b6fc038d424b628b70340cbe9f34ed, entries=150, sequenceid=450, filesize=30.5 K 2024-12-04T19:16:24,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/B/ecdd77d1d2374e438bd9b2dd6b202a8d as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/ecdd77d1d2374e438bd9b2dd6b202a8d 2024-12-04T19:16:24,576 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/ecdd77d1d2374e438bd9b2dd6b202a8d, entries=150, sequenceid=450, filesize=12.0 K 2024-12-04T19:16:24,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/C/becd0bfa340543349ddc8f2e15c8f77d as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/becd0bfa340543349ddc8f2e15c8f77d 2024-12-04T19:16:24,579 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/becd0bfa340543349ddc8f2e15c8f77d, entries=150, sequenceid=450, filesize=12.0 K 2024-12-04T19:16:24,579 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=120.76 KB/123660 for e22a2b2fd059f99265758fd39ea40832 in 852ms, sequenceid=450, compaction requested=true 2024-12-04T19:16:24,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for e22a2b2fd059f99265758fd39ea40832: 2024-12-04T19:16:24,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. 2024-12-04T19:16:24,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-12-04T19:16:24,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-12-04T19:16:24,582 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-12-04T19:16:24,582 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0070 sec 2024-12-04T19:16:24,583 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 1.0100 sec 2024-12-04T19:16:24,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-04T19:16:24,712 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T19:16:24,712 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T19:16:24,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-12-04T19:16:24,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-04T19:16:24,723 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T19:16:24,723 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T19:16:24,723 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T19:16:24,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-04T19:16:24,875 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-04T19:16:24,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. 2024-12-04T19:16:24,876 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing e22a2b2fd059f99265758fd39ea40832 3/3 column families, dataSize=120.76 KB heapSize=340.45 KB 2024-12-04T19:16:24,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e22a2b2fd059f99265758fd39ea40832, store=A 2024-12-04T19:16:24,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T19:16:24,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e22a2b2fd059f99265758fd39ea40832, store=B 2024-12-04T19:16:24,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T19:16:24,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e22a2b2fd059f99265758fd39ea40832, store=C 2024-12-04T19:16:24,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T19:16:24,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412047333ff4a42a24cd4930a0813d661233f_e22a2b2fd059f99265758fd39ea40832 is 50, key is test_row_0/A:col10/1733339783895/Put/seqid=0 2024-12-04T19:16:24,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742402_1578 (size=12454) 2024-12-04T19:16:25,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(8855): Flush requested on e22a2b2fd059f99265758fd39ea40832 2024-12-04T19:16:25,013 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. as already flushing 2024-12-04T19:16:25,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-04T19:16:25,044 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:25,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38946 deadline: 1733339845041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:25,045 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:25,045 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:25,045 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:25,047 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:25,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 234 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38908 deadline: 1733339845044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:25,048 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:25,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38914 deadline: 1733339845045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:25,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:25,048 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:25,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:25,048 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:25,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:25,048 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:25,154 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:25,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 235 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38908 deadline: 1733339845151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:25,154 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:25,154 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:25,154 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:25,154 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:25,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38914 deadline: 1733339845152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:25,154 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:25,154 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:25,154 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:25,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:25,288 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412047333ff4a42a24cd4930a0813d661233f_e22a2b2fd059f99265758fd39ea40832 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412047333ff4a42a24cd4930a0813d661233f_e22a2b2fd059f99265758fd39ea40832 2024-12-04T19:16:25,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/A/36eeafb63b684336ba016d86ab4f66af, store: [table=TestAcidGuarantees family=A region=e22a2b2fd059f99265758fd39ea40832] 2024-12-04T19:16:25,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/A/36eeafb63b684336ba016d86ab4f66af is 175, key is test_row_0/A:col10/1733339783895/Put/seqid=0 2024-12-04T19:16:25,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742403_1579 (size=31255) 2024-12-04T19:16:25,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-04T19:16:25,364 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:25,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 236 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38908 deadline: 1733339845361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:25,364 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:25,364 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:25,364 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:25,365 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:25,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38914 deadline: 1733339845361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:25,365 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:25,365 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:25,365 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:25,674 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:25,674 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:25,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38914 deadline: 1733339845672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:25,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 237 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38908 deadline: 1733339845672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:25,674 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:25,674 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:25,674 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:25,674 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:25,674 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:25,674 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:25,693 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=484, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/A/36eeafb63b684336ba016d86ab4f66af 2024-12-04T19:16:25,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/B/3582ef5bd1e1469f9305b6346d7de84b is 50, key is test_row_0/B:col10/1733339783895/Put/seqid=0 2024-12-04T19:16:25,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742404_1580 (size=12301) 2024-12-04T19:16:25,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-04T19:16:26,052 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:26,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38946 deadline: 1733339846051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:26,052 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to 9ba2734cd39f,33969,1733339629791 for 'test_row_1' in e22a2b2fd059f99265758fd39ea40832 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2147 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-04T19:16:26,053 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:26,053 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:26,053 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:26,102 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=484 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/B/3582ef5bd1e1469f9305b6346d7de84b 2024-12-04T19:16:26,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/C/8f7d3df6a4ae48c8a9e97619f4a82cd1 is 50, key is test_row_0/C:col10/1733339783895/Put/seqid=0 2024-12-04T19:16:26,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742405_1581 (size=12301) 2024-12-04T19:16:26,118 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=484 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/C/8f7d3df6a4ae48c8a9e97619f4a82cd1 2024-12-04T19:16:26,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/A/36eeafb63b684336ba016d86ab4f66af as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/36eeafb63b684336ba016d86ab4f66af 2024-12-04T19:16:26,126 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/36eeafb63b684336ba016d86ab4f66af, entries=150, sequenceid=484, filesize=30.5 K 2024-12-04T19:16:26,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/B/3582ef5bd1e1469f9305b6346d7de84b as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/3582ef5bd1e1469f9305b6346d7de84b 2024-12-04T19:16:26,131 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/3582ef5bd1e1469f9305b6346d7de84b, entries=150, sequenceid=484, filesize=12.0 K 2024-12-04T19:16:26,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/C/8f7d3df6a4ae48c8a9e97619f4a82cd1 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/8f7d3df6a4ae48c8a9e97619f4a82cd1 2024-12-04T19:16:26,137 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/8f7d3df6a4ae48c8a9e97619f4a82cd1, entries=150, sequenceid=484, filesize=12.0 K 2024-12-04T19:16:26,138 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~340.41 KB/348576, currentSize=60.38 KB/61830 for e22a2b2fd059f99265758fd39ea40832 in 1262ms, sequenceid=484, compaction requested=true 2024-12-04T19:16:26,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for e22a2b2fd059f99265758fd39ea40832: 2024-12-04T19:16:26,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. 2024-12-04T19:16:26,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-12-04T19:16:26,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-12-04T19:16:26,142 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-12-04T19:16:26,142 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4170 sec 2024-12-04T19:16:26,145 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 1.4310 sec 2024-12-04T19:16:26,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(8855): Flush requested on e22a2b2fd059f99265758fd39ea40832 2024-12-04T19:16:26,185 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e22a2b2fd059f99265758fd39ea40832 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-04T19:16:26,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e22a2b2fd059f99265758fd39ea40832, store=A 2024-12-04T19:16:26,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:16:26,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e22a2b2fd059f99265758fd39ea40832, store=B 2024-12-04T19:16:26,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:16:26,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e22a2b2fd059f99265758fd39ea40832, store=C 2024-12-04T19:16:26,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:16:26,190 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204998316d32a504546a149751f5cfa269a_e22a2b2fd059f99265758fd39ea40832 is 50, key is test_row_0/A:col10/1733339785032/Put/seqid=0 2024-12-04T19:16:26,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742406_1582 (size=12454) 2024-12-04T19:16:26,223 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:26,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 247 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38908 deadline: 1733339846220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:26,224 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:26,224 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:26,224 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:26,226 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:26,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38914 deadline: 1733339846223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:26,226 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:26,226 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:26,226 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:26,334 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:26,334 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:26,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 248 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38908 deadline: 1733339846331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:26,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38914 deadline: 1733339846331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:26,335 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:26,335 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:26,335 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:26,335 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:26,335 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:26,335 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:26,543 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:26,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 249 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38908 deadline: 1733339846542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:26,546 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:26,546 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:26,546 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:26,546 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:26,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38914 deadline: 1733339846542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:26,547 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:26,547 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:26,547 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:26,593 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:26,596 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204998316d32a504546a149751f5cfa269a_e22a2b2fd059f99265758fd39ea40832 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204998316d32a504546a149751f5cfa269a_e22a2b2fd059f99265758fd39ea40832 2024-12-04T19:16:26,596 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/A/9511eacd509244ad8b58d5eefc0f60fe, store: [table=TestAcidGuarantees family=A region=e22a2b2fd059f99265758fd39ea40832] 2024-12-04T19:16:26,597 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/A/9511eacd509244ad8b58d5eefc0f60fe is 175, key is test_row_0/A:col10/1733339785032/Put/seqid=0 2024-12-04T19:16:26,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742407_1583 (size=31255) 2024-12-04T19:16:26,601 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=498, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/A/9511eacd509244ad8b58d5eefc0f60fe 2024-12-04T19:16:26,606 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/B/73761bfc71274eba974347395a06c1d7 is 50, key is test_row_0/B:col10/1733339785032/Put/seqid=0 2024-12-04T19:16:26,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742408_1584 (size=12301) 2024-12-04T19:16:26,754 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:26,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38922 deadline: 1733339846752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:26,754 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 9ba2734cd39f,33969,1733339629791 for 'test_row_0' in e22a2b2fd059f99265758fd39ea40832 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8211 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-04T19:16:26,755 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:26,755 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:26,755 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:26,773 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:26,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38904 deadline: 1733339846772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:26,773 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 9ba2734cd39f,33969,1733339629791 for 'test_row_2' in e22a2b2fd059f99265758fd39ea40832 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8226 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-04T19:16:26,774 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:26,774 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:26,774 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:26,854 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:26,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 250 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38908 deadline: 1733339846852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:26,854 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:26,854 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:26,854 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:26,854 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:26,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38914 deadline: 1733339846852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:26,855 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:26,855 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:26,855 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:26,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-04T19:16:26,871 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T19:16:26,872 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T19:16:26,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-12-04T19:16:26,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-04T19:16:26,873 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T19:16:26,874 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T19:16:26,874 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T19:16:26,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-04T19:16:27,015 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=498 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/B/73761bfc71274eba974347395a06c1d7 2024-12-04T19:16:27,023 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/C/2c2f9f9708a54e12badd33d38b2c2677 is 50, key is test_row_0/C:col10/1733339785032/Put/seqid=0 2024-12-04T19:16:27,026 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-04T19:16:27,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. 2024-12-04T19:16:27,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. as already flushing 2024-12-04T19:16:27,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. 2024-12-04T19:16:27,026 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:27,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:27,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:27,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742409_1585 (size=12301) 2024-12-04T19:16:27,183 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-04T19:16:27,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. 2024-12-04T19:16:27,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. as already flushing 2024-12-04T19:16:27,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. 2024-12-04T19:16:27,184 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:27,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:27,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:27,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-04T19:16:27,336 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-04T19:16:27,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. 2024-12-04T19:16:27,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. as already flushing 2024-12-04T19:16:27,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. 2024-12-04T19:16:27,336 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:27,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:27,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:27,364 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:27,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 251 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38908 deadline: 1733339847362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:27,365 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:27,365 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:27,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38914 deadline: 1733339847362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:27,365 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:27,365 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:27,365 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:27,365 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:27,365 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:27,448 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=498 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/C/2c2f9f9708a54e12badd33d38b2c2677 2024-12-04T19:16:27,461 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/A/9511eacd509244ad8b58d5eefc0f60fe as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/9511eacd509244ad8b58d5eefc0f60fe 2024-12-04T19:16:27,472 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/9511eacd509244ad8b58d5eefc0f60fe, entries=150, sequenceid=498, filesize=30.5 K 2024-12-04T19:16:27,474 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/B/73761bfc71274eba974347395a06c1d7 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/73761bfc71274eba974347395a06c1d7 2024-12-04T19:16:27,479 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/73761bfc71274eba974347395a06c1d7, entries=150, sequenceid=498, filesize=12.0 K 2024-12-04T19:16:27,481 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/C/2c2f9f9708a54e12badd33d38b2c2677 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/2c2f9f9708a54e12badd33d38b2c2677 2024-12-04T19:16:27,488 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/2c2f9f9708a54e12badd33d38b2c2677, entries=150, sequenceid=498, filesize=12.0 K 2024-12-04T19:16:27,489 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-04T19:16:27,490 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for e22a2b2fd059f99265758fd39ea40832 in 1305ms, sequenceid=498, compaction requested=true 2024-12-04T19:16:27,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. 2024-12-04T19:16:27,490 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e22a2b2fd059f99265758fd39ea40832: 2024-12-04T19:16:27,490 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing e22a2b2fd059f99265758fd39ea40832 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-04T19:16:27,490 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e22a2b2fd059f99265758fd39ea40832:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T19:16:27,490 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T19:16:27,490 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e22a2b2fd059f99265758fd39ea40832:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T19:16:27,490 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-04T19:16:27,490 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e22a2b2fd059f99265758fd39ea40832:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T19:16:27,490 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-12-04T19:16:27,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e22a2b2fd059f99265758fd39ea40832, store=A 2024-12-04T19:16:27,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:16:27,490 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-04T19:16:27,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e22a2b2fd059f99265758fd39ea40832, store=B 2024-12-04T19:16:27,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:16:27,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e22a2b2fd059f99265758fd39ea40832, store=C 2024-12-04T19:16:27,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:16:27,490 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-04T19:16:27,493 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61445 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-04T19:16:27,493 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 173599 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-04T19:16:27,493 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1541): e22a2b2fd059f99265758fd39ea40832/C is initiating minor compaction (all files) 2024-12-04T19:16:27,493 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1541): e22a2b2fd059f99265758fd39ea40832/A is initiating minor compaction (all files) 2024-12-04T19:16:27,493 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e22a2b2fd059f99265758fd39ea40832/C in TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. 2024-12-04T19:16:27,493 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e22a2b2fd059f99265758fd39ea40832/A in TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. 2024-12-04T19:16:27,493 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/fb29d6266a644dec9c81e077a70bccec, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/9f48a10803ae45208609adcf876e1759, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/a2b6fc038d424b628b70340cbe9f34ed, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/36eeafb63b684336ba016d86ab4f66af, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/9511eacd509244ad8b58d5eefc0f60fe] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp, totalSize=169.5 K 2024-12-04T19:16:27,493 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/7c1857b1f5d544d0a6a56699ea2b9e6a, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/5aaea413efe6478e9485ac530987a91f, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/becd0bfa340543349ddc8f2e15c8f77d, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/8f7d3df6a4ae48c8a9e97619f4a82cd1, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/2c2f9f9708a54e12badd33d38b2c2677] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp, totalSize=60.0 K 2024-12-04T19:16:27,493 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=11 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. 2024-12-04T19:16:27,493 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. files: [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/fb29d6266a644dec9c81e077a70bccec, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/9f48a10803ae45208609adcf876e1759, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/a2b6fc038d424b628b70340cbe9f34ed, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/36eeafb63b684336ba016d86ab4f66af, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/9511eacd509244ad8b58d5eefc0f60fe] 2024-12-04T19:16:27,494 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c1857b1f5d544d0a6a56699ea2b9e6a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1733339780643 2024-12-04T19:16:27,494 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting fb29d6266a644dec9c81e077a70bccec, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1733339780643 2024-12-04T19:16:27,494 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 5aaea413efe6478e9485ac530987a91f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=435, earliestPutTs=1733339782673 2024-12-04T19:16:27,494 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9f48a10803ae45208609adcf876e1759, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=435, earliestPutTs=1733339782673 2024-12-04T19:16:27,495 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting becd0bfa340543349ddc8f2e15c8f77d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=450, earliestPutTs=1733339782702 2024-12-04T19:16:27,495 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting a2b6fc038d424b628b70340cbe9f34ed, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=450, earliestPutTs=1733339782702 2024-12-04T19:16:27,495 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f7d3df6a4ae48c8a9e97619f4a82cd1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=484, earliestPutTs=1733339783895 2024-12-04T19:16:27,495 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c2f9f9708a54e12badd33d38b2c2677, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=498, earliestPutTs=1733339785032 2024-12-04T19:16:27,495 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 36eeafb63b684336ba016d86ab4f66af, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=484, earliestPutTs=1733339783895 2024-12-04T19:16:27,496 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9511eacd509244ad8b58d5eefc0f60fe, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=498, earliestPutTs=1733339785032 2024-12-04T19:16:27,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204c2b0560abcc244939cecdada755ee168_e22a2b2fd059f99265758fd39ea40832 is 50, key is test_row_0/A:col10/1733339786222/Put/seqid=0 2024-12-04T19:16:27,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742410_1586 (size=12454) 2024-12-04T19:16:27,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:27,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-04T19:16:27,505 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204c2b0560abcc244939cecdada755ee168_e22a2b2fd059f99265758fd39ea40832 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204c2b0560abcc244939cecdada755ee168_e22a2b2fd059f99265758fd39ea40832 2024-12-04T19:16:27,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/A/d3619cac95174c1ca6ffe45027cb97e9, store: [table=TestAcidGuarantees family=A region=e22a2b2fd059f99265758fd39ea40832] 2024-12-04T19:16:27,506 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e22a2b2fd059f99265758fd39ea40832#C#compaction#495 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:16:27,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/A/d3619cac95174c1ca6ffe45027cb97e9 is 175, key is test_row_0/A:col10/1733339786222/Put/seqid=0 2024-12-04T19:16:27,506 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/C/ac4a4ebc2d5b48379c26adaa84721500 is 50, key is test_row_0/C:col10/1733339785032/Put/seqid=0 2024-12-04T19:16:27,507 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e22a2b2fd059f99265758fd39ea40832] 2024-12-04T19:16:27,510 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241204b592cddb0637427094f4ea12f5798ea2_e22a2b2fd059f99265758fd39ea40832 store=[table=TestAcidGuarantees family=A region=e22a2b2fd059f99265758fd39ea40832] 2024-12-04T19:16:27,511 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241204b592cddb0637427094f4ea12f5798ea2_e22a2b2fd059f99265758fd39ea40832, store=[table=TestAcidGuarantees family=A region=e22a2b2fd059f99265758fd39ea40832] 2024-12-04T19:16:27,511 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204b592cddb0637427094f4ea12f5798ea2_e22a2b2fd059f99265758fd39ea40832 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e22a2b2fd059f99265758fd39ea40832] 2024-12-04T19:16:27,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742411_1587 (size=31255) 2024-12-04T19:16:27,514 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=516, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/A/d3619cac95174c1ca6ffe45027cb97e9 2024-12-04T19:16:27,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742412_1588 (size=12711) 2024-12-04T19:16:27,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742413_1589 (size=4469) 2024-12-04T19:16:27,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/B/e795285d9ba348d4ad6c9ffc5590c8ab is 50, key is test_row_0/B:col10/1733339786222/Put/seqid=0 2024-12-04T19:16:27,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742414_1590 (size=12301) 2024-12-04T19:16:27,582 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=516 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/B/e795285d9ba348d4ad6c9ffc5590c8ab 2024-12-04T19:16:27,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/C/dbf23d703a7349e4b31a58cc0423175d is 50, key is test_row_0/C:col10/1733339786222/Put/seqid=0 2024-12-04T19:16:27,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742415_1591 (size=12301) 2024-12-04T19:16:27,924 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/C/ac4a4ebc2d5b48379c26adaa84721500 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/ac4a4ebc2d5b48379c26adaa84721500 2024-12-04T19:16:27,927 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in e22a2b2fd059f99265758fd39ea40832/C of e22a2b2fd059f99265758fd39ea40832 into ac4a4ebc2d5b48379c26adaa84721500(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:16:27,927 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e22a2b2fd059f99265758fd39ea40832: 2024-12-04T19:16:27,927 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., storeName=e22a2b2fd059f99265758fd39ea40832/C, priority=11, startTime=1733339787490; duration=0sec 2024-12-04T19:16:27,927 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T19:16:27,927 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e22a2b2fd059f99265758fd39ea40832:C 2024-12-04T19:16:27,927 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-04T19:16:27,928 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61445 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-04T19:16:27,928 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1541): e22a2b2fd059f99265758fd39ea40832/B is initiating minor compaction (all files) 2024-12-04T19:16:27,928 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e22a2b2fd059f99265758fd39ea40832/B in TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. 2024-12-04T19:16:27,928 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/99b16982d0a4470bb478a045ef63b33f, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/785e4c4c9f7e4d91975486886083afae, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/ecdd77d1d2374e438bd9b2dd6b202a8d, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/3582ef5bd1e1469f9305b6346d7de84b, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/73761bfc71274eba974347395a06c1d7] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp, totalSize=60.0 K 2024-12-04T19:16:27,928 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 99b16982d0a4470bb478a045ef63b33f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1733339780643 2024-12-04T19:16:27,929 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 785e4c4c9f7e4d91975486886083afae, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=435, earliestPutTs=1733339782673 2024-12-04T19:16:27,929 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting ecdd77d1d2374e438bd9b2dd6b202a8d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=450, earliestPutTs=1733339782702 2024-12-04T19:16:27,929 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 3582ef5bd1e1469f9305b6346d7de84b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=484, earliestPutTs=1733339783895 2024-12-04T19:16:27,929 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 73761bfc71274eba974347395a06c1d7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=498, earliestPutTs=1733339785032 2024-12-04T19:16:27,935 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e22a2b2fd059f99265758fd39ea40832#A#compaction#496 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:16:27,935 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/A/f53eb536d0334201b3de728485a691a7 is 175, key is test_row_0/A:col10/1733339785032/Put/seqid=0 2024-12-04T19:16:27,936 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e22a2b2fd059f99265758fd39ea40832#B#compaction#499 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:16:27,936 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/B/0c7ee0ee94d4462383e9187079fb85a6 is 50, key is test_row_0/B:col10/1733339785032/Put/seqid=0 2024-12-04T19:16:27,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742417_1593 (size=12711) 2024-12-04T19:16:27,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742416_1592 (size=31665) 2024-12-04T19:16:27,943 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/A/f53eb536d0334201b3de728485a691a7 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/f53eb536d0334201b3de728485a691a7 2024-12-04T19:16:27,947 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in e22a2b2fd059f99265758fd39ea40832/A of e22a2b2fd059f99265758fd39ea40832 into f53eb536d0334201b3de728485a691a7(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:16:27,947 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e22a2b2fd059f99265758fd39ea40832: 2024-12-04T19:16:27,947 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., storeName=e22a2b2fd059f99265758fd39ea40832/A, priority=11, startTime=1733339787490; duration=0sec 2024-12-04T19:16:27,947 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:16:27,947 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e22a2b2fd059f99265758fd39ea40832:A 2024-12-04T19:16:28,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-04T19:16:28,019 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=516 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/C/dbf23d703a7349e4b31a58cc0423175d 2024-12-04T19:16:28,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/A/d3619cac95174c1ca6ffe45027cb97e9 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/d3619cac95174c1ca6ffe45027cb97e9 2024-12-04T19:16:28,025 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/d3619cac95174c1ca6ffe45027cb97e9, entries=150, sequenceid=516, filesize=30.5 K 2024-12-04T19:16:28,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/B/e795285d9ba348d4ad6c9ffc5590c8ab as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/e795285d9ba348d4ad6c9ffc5590c8ab 2024-12-04T19:16:28,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,028 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/e795285d9ba348d4ad6c9ffc5590c8ab, entries=150, sequenceid=516, filesize=12.0 K 2024-12-04T19:16:28,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/C/dbf23d703a7349e4b31a58cc0423175d as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/dbf23d703a7349e4b31a58cc0423175d 2024-12-04T19:16:28,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,032 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/dbf23d703a7349e4b31a58cc0423175d, entries=150, sequenceid=516, filesize=12.0 K 2024-12-04T19:16:28,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,033 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=0 B/0 for e22a2b2fd059f99265758fd39ea40832 in 543ms, sequenceid=516, compaction requested=false 2024-12-04T19:16:28,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for e22a2b2fd059f99265758fd39ea40832: 2024-12-04T19:16:28,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. 2024-12-04T19:16:28,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-12-04T19:16:28,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-12-04T19:16:28,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,036 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-12-04T19:16:28,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,036 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1600 sec 2024-12-04T19:16:28,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,038 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 1.1650 sec 2024-12-04T19:16:28,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(8855): Flush requested on e22a2b2fd059f99265758fd39ea40832 2024-12-04T19:16:28,128 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e22a2b2fd059f99265758fd39ea40832 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-04T19:16:28,128 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e22a2b2fd059f99265758fd39ea40832, store=A 2024-12-04T19:16:28,128 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:16:28,128 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e22a2b2fd059f99265758fd39ea40832, store=B 2024-12-04T19:16:28,128 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:16:28,128 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e22a2b2fd059f99265758fd39ea40832, store=C 2024-12-04T19:16:28,128 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:16:28,133 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412045f1b538192ce47d3aa56fef69c439f11_e22a2b2fd059f99265758fd39ea40832 is 50, key is test_row_0/A:col10/1733339788109/Put/seqid=0 2024-12-04T19:16:28,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742418_1594 (size=14994) 2024-12-04T19:16:28,139 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:28,144 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412045f1b538192ce47d3aa56fef69c439f11_e22a2b2fd059f99265758fd39ea40832 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412045f1b538192ce47d3aa56fef69c439f11_e22a2b2fd059f99265758fd39ea40832 2024-12-04T19:16:28,153 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/A/2761006941da4859bb159b5301ba594e, store: [table=TestAcidGuarantees family=A region=e22a2b2fd059f99265758fd39ea40832] 2024-12-04T19:16:28,154 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/A/2761006941da4859bb159b5301ba594e is 175, key is test_row_0/A:col10/1733339788109/Put/seqid=0 2024-12-04T19:16:28,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742419_1595 (size=39949) 2024-12-04T19:16:28,238 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733339788214, max timestamp=1733339788235Num uniques 859; 2024-12-04T19:16:28,238 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733339788214, max timestamp=1733339788235Num uniques 859; 2024-12-04T19:16:28,238 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733339788214, max timestamp=1733339788235Num uniques 859; 2024-12-04T19:16:28,238 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T19:16:28,238 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T19:16:28,238 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T19:16:28,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:28,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 253 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38946 deadline: 1733339848318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:28,322 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:28,322 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:28,322 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:28,342 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/B/0c7ee0ee94d4462383e9187079fb85a6 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/0c7ee0ee94d4462383e9187079fb85a6 2024-12-04T19:16:28,345 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in e22a2b2fd059f99265758fd39ea40832/B of e22a2b2fd059f99265758fd39ea40832 into 0c7ee0ee94d4462383e9187079fb85a6(size=12.4 K), total size for store is 24.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:16:28,345 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e22a2b2fd059f99265758fd39ea40832: 2024-12-04T19:16:28,345 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., storeName=e22a2b2fd059f99265758fd39ea40832/B, priority=11, startTime=1733339787490; duration=0sec 2024-12-04T19:16:28,345 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:16:28,345 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e22a2b2fd059f99265758fd39ea40832:B 2024-12-04T19:16:28,373 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:28,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38914 deadline: 1733339848372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:28,373 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 9ba2734cd39f,33969,1733339629791 for 'test_row_0' in e22a2b2fd059f99265758fd39ea40832 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2150 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-04T19:16:28,374 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:28,374 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:28,374 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:28,384 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:28,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 252 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38908 deadline: 1733339848382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:28,384 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 9ba2734cd39f,33969,1733339629791 for 'test_row_2' in e22a2b2fd059f99265758fd39ea40832 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-04T19:16:28,385 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:28,385 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:28,385 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:28,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:28,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 254 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38946 deadline: 1733339848431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:28,434 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:28,434 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:28,434 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:28,595 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=528, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/A/2761006941da4859bb159b5301ba594e 2024-12-04T19:16:28,600 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/B/d6d0599d7c3c470788ace562699b81ee is 50, key is test_row_0/B:col10/1733339788109/Put/seqid=0 2024-12-04T19:16:28,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742420_1596 (size=12301) 2024-12-04T19:16:28,644 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:28,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 255 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38946 deadline: 1733339848642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:28,645 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:28,645 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:28,645 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:28,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:28,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 256 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38946 deadline: 1733339848951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:28,952 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:28,952 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:28,952 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:29,003 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=528 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/B/d6d0599d7c3c470788ace562699b81ee 2024-12-04T19:16:29,010 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/C/48a746ee70c0450e8f1196c82d6ec8b9 is 50, key is test_row_0/C:col10/1733339788109/Put/seqid=0 2024-12-04T19:16:29,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742421_1597 (size=12301) 2024-12-04T19:16:29,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-04T19:16:29,022 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T19:16:29,022 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T19:16:29,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-12-04T19:16:29,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-04T19:16:29,024 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T19:16:29,024 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T19:16:29,024 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T19:16:29,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-04T19:16:29,176 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-04T19:16:29,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. 2024-12-04T19:16:29,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. as already flushing 2024-12-04T19:16:29,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. 2024-12-04T19:16:29,177 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:29,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:29,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:29,329 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-04T19:16:29,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. 2024-12-04T19:16:29,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. as already flushing 2024-12-04T19:16:29,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. 2024-12-04T19:16:29,329 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:29,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:29,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:29,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-04T19:16:29,417 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=528 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/C/48a746ee70c0450e8f1196c82d6ec8b9 2024-12-04T19:16:29,420 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/A/2761006941da4859bb159b5301ba594e as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/2761006941da4859bb159b5301ba594e 2024-12-04T19:16:29,423 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/2761006941da4859bb159b5301ba594e, entries=200, sequenceid=528, filesize=39.0 K 2024-12-04T19:16:29,423 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/B/d6d0599d7c3c470788ace562699b81ee as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/d6d0599d7c3c470788ace562699b81ee 2024-12-04T19:16:29,426 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/d6d0599d7c3c470788ace562699b81ee, entries=150, sequenceid=528, filesize=12.0 K 2024-12-04T19:16:29,426 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(997): StoreScanner already has the close lock. There is no need to updateReaders 2024-12-04T19:16:29,426 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/C/48a746ee70c0450e8f1196c82d6ec8b9 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/48a746ee70c0450e8f1196c82d6ec8b9 2024-12-04T19:16:29,429 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/48a746ee70c0450e8f1196c82d6ec8b9, entries=150, sequenceid=528, filesize=12.0 K 2024-12-04T19:16:29,430 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for e22a2b2fd059f99265758fd39ea40832 in 1301ms, sequenceid=528, compaction requested=true 2024-12-04T19:16:29,430 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e22a2b2fd059f99265758fd39ea40832: 2024-12-04T19:16:29,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e22a2b2fd059f99265758fd39ea40832:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T19:16:29,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:16:29,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e22a2b2fd059f99265758fd39ea40832:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T19:16:29,430 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T19:16:29,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:16:29,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e22a2b2fd059f99265758fd39ea40832:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T19:16:29,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T19:16:29,430 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T19:16:29,430 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37313 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T19:16:29,430 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102869 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T19:16:29,430 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1541): e22a2b2fd059f99265758fd39ea40832/A is initiating minor compaction (all files) 2024-12-04T19:16:29,430 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1541): e22a2b2fd059f99265758fd39ea40832/B is initiating minor compaction (all files) 2024-12-04T19:16:29,430 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e22a2b2fd059f99265758fd39ea40832/B in TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. 2024-12-04T19:16:29,430 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e22a2b2fd059f99265758fd39ea40832/A in TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. 2024-12-04T19:16:29,431 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/0c7ee0ee94d4462383e9187079fb85a6, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/e795285d9ba348d4ad6c9ffc5590c8ab, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/d6d0599d7c3c470788ace562699b81ee] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp, totalSize=36.4 K 2024-12-04T19:16:29,431 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/f53eb536d0334201b3de728485a691a7, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/d3619cac95174c1ca6ffe45027cb97e9, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/2761006941da4859bb159b5301ba594e] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp, totalSize=100.5 K 2024-12-04T19:16:29,431 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. 2024-12-04T19:16:29,431 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. files: [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/f53eb536d0334201b3de728485a691a7, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/d3619cac95174c1ca6ffe45027cb97e9, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/2761006941da4859bb159b5301ba594e] 2024-12-04T19:16:29,431 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c7ee0ee94d4462383e9187079fb85a6, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=498, earliestPutTs=1733339785032 2024-12-04T19:16:29,431 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting f53eb536d0334201b3de728485a691a7, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=498, earliestPutTs=1733339785032 2024-12-04T19:16:29,431 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting e795285d9ba348d4ad6c9ffc5590c8ab, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=516, earliestPutTs=1733339786212 2024-12-04T19:16:29,431 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting d3619cac95174c1ca6ffe45027cb97e9, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=516, earliestPutTs=1733339786212 2024-12-04T19:16:29,431 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting d6d0599d7c3c470788ace562699b81ee, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=528, earliestPutTs=1733339788109 2024-12-04T19:16:29,431 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2761006941da4859bb159b5301ba594e, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=528, earliestPutTs=1733339788109 2024-12-04T19:16:29,436 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e22a2b2fd059f99265758fd39ea40832#B#compaction#503 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:16:29,437 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/B/bea39d43bde142b9b4fe87664db4b9d6 is 50, key is test_row_0/B:col10/1733339788109/Put/seqid=0 2024-12-04T19:16:29,437 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e22a2b2fd059f99265758fd39ea40832] 2024-12-04T19:16:29,442 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120465bc86fbdee34db1acc6bb1b144d4f0a_e22a2b2fd059f99265758fd39ea40832 store=[table=TestAcidGuarantees family=A region=e22a2b2fd059f99265758fd39ea40832] 2024-12-04T19:16:29,443 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120465bc86fbdee34db1acc6bb1b144d4f0a_e22a2b2fd059f99265758fd39ea40832, store=[table=TestAcidGuarantees family=A region=e22a2b2fd059f99265758fd39ea40832] 2024-12-04T19:16:29,443 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120465bc86fbdee34db1acc6bb1b144d4f0a_e22a2b2fd059f99265758fd39ea40832 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e22a2b2fd059f99265758fd39ea40832] 2024-12-04T19:16:29,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742422_1598 (size=12813) 2024-12-04T19:16:29,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742423_1599 (size=4469) 2024-12-04T19:16:29,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(8855): Flush requested on e22a2b2fd059f99265758fd39ea40832 2024-12-04T19:16:29,464 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e22a2b2fd059f99265758fd39ea40832 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-12-04T19:16:29,464 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e22a2b2fd059f99265758fd39ea40832, store=A 2024-12-04T19:16:29,464 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-04T19:16:29,464 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e22a2b2fd059f99265758fd39ea40832, store=B 2024-12-04T19:16:29,464 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-04T19:16:29,464 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e22a2b2fd059f99265758fd39ea40832, store=C 2024-12-04T19:16:29,464 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-04T19:16:29,472 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204d88f62edc2114bcc9f355174e75dfd2c_e22a2b2fd059f99265758fd39ea40832 is 50, key is test_row_0/A:col10/1733339788302/Put/seqid=0 2024-12-04T19:16:29,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742424_1600 (size=14994) 2024-12-04T19:16:29,475 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:29,481 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-04T19:16:29,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. 2024-12-04T19:16:29,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. as already flushing 2024-12-04T19:16:29,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. 2024-12-04T19:16:29,482 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:29,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:29,482 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204d88f62edc2114bcc9f355174e75dfd2c_e22a2b2fd059f99265758fd39ea40832 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204d88f62edc2114bcc9f355174e75dfd2c_e22a2b2fd059f99265758fd39ea40832 2024-12-04T19:16:29,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:29,483 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/A/12dff66832ba4d97bf39771d440185e9, store: [table=TestAcidGuarantees family=A region=e22a2b2fd059f99265758fd39ea40832] 2024-12-04T19:16:29,484 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/A/12dff66832ba4d97bf39771d440185e9 is 175, key is test_row_0/A:col10/1733339788302/Put/seqid=0 2024-12-04T19:16:29,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742425_1601 (size=39949) 2024-12-04T19:16:29,491 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:29,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 264 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38946 deadline: 1733339849488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:29,492 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:29,492 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:29,492 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:29,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:29,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 265 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38946 deadline: 1733339849601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:29,605 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:29,605 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:29,605 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:29,636 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-04T19:16:29,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. 2024-12-04T19:16:29,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. as already flushing 2024-12-04T19:16:29,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. 2024-12-04T19:16:29,636 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:29,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:29,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:29,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-04T19:16:29,794 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-04T19:16:29,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. 2024-12-04T19:16:29,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. as already flushing 2024-12-04T19:16:29,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. 2024-12-04T19:16:29,794 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:29,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:29,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:29,814 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:29,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 266 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38946 deadline: 1733339849812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:29,814 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:29,814 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:29,814 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:29,851 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e22a2b2fd059f99265758fd39ea40832#A#compaction#504 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:16:29,852 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/A/e000a16457964f44aaea140c421f9227 is 175, key is test_row_0/A:col10/1733339788109/Put/seqid=0 2024-12-04T19:16:29,852 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/B/bea39d43bde142b9b4fe87664db4b9d6 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/bea39d43bde142b9b4fe87664db4b9d6 2024-12-04T19:16:29,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742426_1602 (size=31767) 2024-12-04T19:16:29,856 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e22a2b2fd059f99265758fd39ea40832/B of e22a2b2fd059f99265758fd39ea40832 into bea39d43bde142b9b4fe87664db4b9d6(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:16:29,856 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e22a2b2fd059f99265758fd39ea40832: 2024-12-04T19:16:29,856 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., storeName=e22a2b2fd059f99265758fd39ea40832/B, priority=13, startTime=1733339789430; duration=0sec 2024-12-04T19:16:29,856 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T19:16:29,857 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e22a2b2fd059f99265758fd39ea40832:B 2024-12-04T19:16:29,857 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T19:16:29,857 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37313 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T19:16:29,857 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1541): e22a2b2fd059f99265758fd39ea40832/C is initiating minor compaction (all files) 2024-12-04T19:16:29,857 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e22a2b2fd059f99265758fd39ea40832/C in TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. 2024-12-04T19:16:29,857 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/ac4a4ebc2d5b48379c26adaa84721500, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/dbf23d703a7349e4b31a58cc0423175d, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/48a746ee70c0450e8f1196c82d6ec8b9] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp, totalSize=36.4 K 2024-12-04T19:16:29,858 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting ac4a4ebc2d5b48379c26adaa84721500, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=498, earliestPutTs=1733339785032 2024-12-04T19:16:29,858 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting dbf23d703a7349e4b31a58cc0423175d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=516, earliestPutTs=1733339786212 2024-12-04T19:16:29,858 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 48a746ee70c0450e8f1196c82d6ec8b9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=528, earliestPutTs=1733339788109 2024-12-04T19:16:29,870 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e22a2b2fd059f99265758fd39ea40832#C#compaction#506 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:16:29,870 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/C/37eaac5afbed447ca4c8e176a7a568ad is 50, key is test_row_0/C:col10/1733339788109/Put/seqid=0 2024-12-04T19:16:29,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742427_1603 (size=12813) 2024-12-04T19:16:29,887 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=567, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/A/12dff66832ba4d97bf39771d440185e9 2024-12-04T19:16:29,892 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/B/fdffdd3f750c4cd596210f0a7a958034 is 50, key is test_row_0/B:col10/1733339788302/Put/seqid=0 2024-12-04T19:16:29,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742428_1604 (size=12301) 2024-12-04T19:16:29,946 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-04T19:16:29,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. 2024-12-04T19:16:29,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. as already flushing 2024-12-04T19:16:29,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. 2024-12-04T19:16:29,946 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:29,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:29,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:30,099 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-04T19:16:30,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. 2024-12-04T19:16:30,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. as already flushing 2024-12-04T19:16:30,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. 2024-12-04T19:16:30,099 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:30,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:30,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:30,123 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:30,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 267 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38946 deadline: 1733339850121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:30,124 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:30,124 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:30,124 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:30,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-04T19:16:30,251 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-04T19:16:30,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. 2024-12-04T19:16:30,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. as already flushing 2024-12-04T19:16:30,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. 2024-12-04T19:16:30,252 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:30,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:30,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:30,258 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/A/e000a16457964f44aaea140c421f9227 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/e000a16457964f44aaea140c421f9227 2024-12-04T19:16:30,260 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e22a2b2fd059f99265758fd39ea40832/A of e22a2b2fd059f99265758fd39ea40832 into e000a16457964f44aaea140c421f9227(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:16:30,260 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e22a2b2fd059f99265758fd39ea40832: 2024-12-04T19:16:30,260 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., storeName=e22a2b2fd059f99265758fd39ea40832/A, priority=13, startTime=1733339789430; duration=0sec 2024-12-04T19:16:30,260 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:16:30,260 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e22a2b2fd059f99265758fd39ea40832:A 2024-12-04T19:16:30,277 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/C/37eaac5afbed447ca4c8e176a7a568ad as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/37eaac5afbed447ca4c8e176a7a568ad 2024-12-04T19:16:30,280 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e22a2b2fd059f99265758fd39ea40832/C of e22a2b2fd059f99265758fd39ea40832 into 37eaac5afbed447ca4c8e176a7a568ad(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:16:30,280 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e22a2b2fd059f99265758fd39ea40832: 2024-12-04T19:16:30,280 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., storeName=e22a2b2fd059f99265758fd39ea40832/C, priority=13, startTime=1733339789430; duration=0sec 2024-12-04T19:16:30,280 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:16:30,280 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e22a2b2fd059f99265758fd39ea40832:C 2024-12-04T19:16:30,296 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=567 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/B/fdffdd3f750c4cd596210f0a7a958034 2024-12-04T19:16:30,301 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/C/c53448bd0f554ff8b64dc65c8785b77f is 50, key is test_row_0/C:col10/1733339788302/Put/seqid=0 2024-12-04T19:16:30,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742429_1605 (size=12301) 2024-12-04T19:16:30,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:30,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38914 deadline: 1733339850391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:30,395 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 9ba2734cd39f,33969,1733339629791 for 'test_row_0' in e22a2b2fd059f99265758fd39ea40832 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-04T19:16:30,395 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:30,395 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:30,395 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:30,404 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-04T19:16:30,404 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:30,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. 2024-12-04T19:16:30,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. as already flushing 2024-12-04T19:16:30,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 253 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38908 deadline: 1733339850401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:30,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. 2024-12-04T19:16:30,404 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:30,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:30,404 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 9ba2734cd39f,33969,1733339629791 for 'test_row_2' in e22a2b2fd059f99265758fd39ea40832 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-04T19:16:30,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:30,405 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:30,405 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:30,405 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:30,556 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-04T19:16:30,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. 2024-12-04T19:16:30,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. as already flushing 2024-12-04T19:16:30,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. 2024-12-04T19:16:30,557 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:30,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:30,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:30,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:30,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 268 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38946 deadline: 1733339850631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:30,635 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:30,635 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:30,635 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:30,704 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=567 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/C/c53448bd0f554ff8b64dc65c8785b77f 2024-12-04T19:16:30,707 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/A/12dff66832ba4d97bf39771d440185e9 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/12dff66832ba4d97bf39771d440185e9 2024-12-04T19:16:30,709 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-04T19:16:30,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. 2024-12-04T19:16:30,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. as already flushing 2024-12-04T19:16:30,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. 2024-12-04T19:16:30,709 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/12dff66832ba4d97bf39771d440185e9, entries=200, sequenceid=567, filesize=39.0 K 2024-12-04T19:16:30,709 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:30,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:30,710 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/B/fdffdd3f750c4cd596210f0a7a958034 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/fdffdd3f750c4cd596210f0a7a958034 2024-12-04T19:16:30,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:30,712 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/fdffdd3f750c4cd596210f0a7a958034, entries=150, sequenceid=567, filesize=12.0 K 2024-12-04T19:16:30,713 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/C/c53448bd0f554ff8b64dc65c8785b77f as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/c53448bd0f554ff8b64dc65c8785b77f 2024-12-04T19:16:30,715 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/c53448bd0f554ff8b64dc65c8785b77f, entries=150, sequenceid=567, filesize=12.0 K 2024-12-04T19:16:30,716 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~405.56 KB/415296, currentSize=40.25 KB/41220 for e22a2b2fd059f99265758fd39ea40832 in 1253ms, sequenceid=567, compaction requested=false 2024-12-04T19:16:30,716 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e22a2b2fd059f99265758fd39ea40832: 2024-12-04T19:16:30,861 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-04T19:16:30,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. 2024-12-04T19:16:30,862 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing e22a2b2fd059f99265758fd39ea40832 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-12-04T19:16:30,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e22a2b2fd059f99265758fd39ea40832, store=A 2024-12-04T19:16:30,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:16:30,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e22a2b2fd059f99265758fd39ea40832, store=B 2024-12-04T19:16:30,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:16:30,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e22a2b2fd059f99265758fd39ea40832, store=C 2024-12-04T19:16:30,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:16:30,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204d092d7597a2449f4bf8a419c8ea9e67a_e22a2b2fd059f99265758fd39ea40832 is 50, key is test_row_0/A:col10/1733339789482/Put/seqid=0 2024-12-04T19:16:30,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742430_1606 (size=12454) 2024-12-04T19:16:31,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-04T19:16:31,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:31,272 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204d092d7597a2449f4bf8a419c8ea9e67a_e22a2b2fd059f99265758fd39ea40832 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204d092d7597a2449f4bf8a419c8ea9e67a_e22a2b2fd059f99265758fd39ea40832 2024-12-04T19:16:31,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/A/4c7a9594254c4145bb57b14ead897576, store: [table=TestAcidGuarantees family=A region=e22a2b2fd059f99265758fd39ea40832] 2024-12-04T19:16:31,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/A/4c7a9594254c4145bb57b14ead897576 is 175, key is test_row_0/A:col10/1733339789482/Put/seqid=0 2024-12-04T19:16:31,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742431_1607 (size=31255) 2024-12-04T19:16:31,644 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. as already flushing 2024-12-04T19:16:31,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(8855): Flush requested on e22a2b2fd059f99265758fd39ea40832 2024-12-04T19:16:31,677 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=579, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/A/4c7a9594254c4145bb57b14ead897576 2024-12-04T19:16:31,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/B/577dffbcf3924a3db7a3feac0a4361f5 is 50, key is test_row_0/B:col10/1733339789482/Put/seqid=0 2024-12-04T19:16:31,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742432_1608 (size=12301) 2024-12-04T19:16:31,687 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=579 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/B/577dffbcf3924a3db7a3feac0a4361f5 2024-12-04T19:16:31,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/C/d83a9484443546298b44a592d1a0ba97 is 50, key is test_row_0/C:col10/1733339789482/Put/seqid=0 2024-12-04T19:16:31,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742433_1609 (size=12301) 2024-12-04T19:16:31,695 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=579 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/C/d83a9484443546298b44a592d1a0ba97 2024-12-04T19:16:31,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/A/4c7a9594254c4145bb57b14ead897576 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/4c7a9594254c4145bb57b14ead897576 2024-12-04T19:16:31,702 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/4c7a9594254c4145bb57b14ead897576, entries=150, sequenceid=579, filesize=30.5 K 2024-12-04T19:16:31,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/B/577dffbcf3924a3db7a3feac0a4361f5 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/577dffbcf3924a3db7a3feac0a4361f5 2024-12-04T19:16:31,707 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/577dffbcf3924a3db7a3feac0a4361f5, entries=150, sequenceid=579, filesize=12.0 K 2024-12-04T19:16:31,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/C/d83a9484443546298b44a592d1a0ba97 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/d83a9484443546298b44a592d1a0ba97 2024-12-04T19:16:31,711 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733339791694, max timestamp=1733339791708Num uniques 859; 2024-12-04T19:16:31,711 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T19:16:31,711 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733339791694, max timestamp=1733339791708Num uniques 859; 2024-12-04T19:16:31,711 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733339791694, max timestamp=1733339791708Num uniques 859; 2024-12-04T19:16:31,711 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T19:16:31,711 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T19:16:31,715 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/d83a9484443546298b44a592d1a0ba97, entries=150, sequenceid=579, filesize=12.0 K 2024-12-04T19:16:31,723 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=33.54 KB/34350 for e22a2b2fd059f99265758fd39ea40832 in 861ms, sequenceid=579, compaction requested=true 2024-12-04T19:16:31,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for e22a2b2fd059f99265758fd39ea40832: 2024-12-04T19:16:31,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. 2024-12-04T19:16:31,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-12-04T19:16:31,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-12-04T19:16:31,725 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-12-04T19:16:31,725 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7000 sec 2024-12-04T19:16:31,726 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 2.7030 sec 2024-12-04T19:16:31,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(8855): Flush requested on e22a2b2fd059f99265758fd39ea40832 2024-12-04T19:16:31,728 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e22a2b2fd059f99265758fd39ea40832 3/3 column families, dataSize=53.67 KB heapSize=145.92 KB 2024-12-04T19:16:31,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e22a2b2fd059f99265758fd39ea40832, store=A 2024-12-04T19:16:31,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T19:16:31,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e22a2b2fd059f99265758fd39ea40832, store=B 2024-12-04T19:16:31,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T19:16:31,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e22a2b2fd059f99265758fd39ea40832, store=C 2024-12-04T19:16:31,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T19:16:31,732 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412047cf1f19b4898407d91d29a31b65b1535_e22a2b2fd059f99265758fd39ea40832 is 50, key is test_row_0/A:col10/1733339791713/Put/seqid=0 2024-12-04T19:16:31,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742434_1610 (size=12454) 2024-12-04T19:16:31,798 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733339791783, max timestamp=1733339791796Num uniques 859; 2024-12-04T19:16:31,798 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T19:16:31,798 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733339791783, max timestamp=1733339791796Num uniques 859; 2024-12-04T19:16:31,798 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T19:16:31,798 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733339791783, max timestamp=1733339791796Num uniques 859; 2024-12-04T19:16:31,799 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T19:16:31,869 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:31,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 325 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38946 deadline: 1733339851865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:31,869 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:31,869 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:31,869 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:31,973 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:31,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 326 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38946 deadline: 1733339851971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:31,973 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:31,973 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:31,973 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:32,136 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:32,138 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412047cf1f19b4898407d91d29a31b65b1535_e22a2b2fd059f99265758fd39ea40832 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412047cf1f19b4898407d91d29a31b65b1535_e22a2b2fd059f99265758fd39ea40832 2024-12-04T19:16:32,138 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/A/d2efbcb8b45442629764064cba6f4c5b, store: [table=TestAcidGuarantees family=A region=e22a2b2fd059f99265758fd39ea40832] 2024-12-04T19:16:32,139 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/A/d2efbcb8b45442629764064cba6f4c5b is 175, key is test_row_0/A:col10/1733339791713/Put/seqid=0 2024-12-04T19:16:32,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742435_1611 (size=31255) 2024-12-04T19:16:32,142 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=604, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/A/d2efbcb8b45442629764064cba6f4c5b 2024-12-04T19:16:32,147 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/B/3e57124039ea413ca506424b8097a237 is 50, key is test_row_0/B:col10/1733339791713/Put/seqid=0 2024-12-04T19:16:32,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742436_1612 (size=12301) 2024-12-04T19:16:32,149 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=604 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/B/3e57124039ea413ca506424b8097a237 2024-12-04T19:16:32,153 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/C/bbb108a1f633472093f816f62ae77c4e is 50, key is test_row_0/C:col10/1733339791713/Put/seqid=0 2024-12-04T19:16:32,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742437_1613 (size=12301) 2024-12-04T19:16:32,184 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:32,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 327 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38946 deadline: 1733339852182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:32,185 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:32,185 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:32,185 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:32,496 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:32,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 328 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38946 deadline: 1733339852492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:32,496 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:32,496 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:32,496 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:32,556 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=604 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/C/bbb108a1f633472093f816f62ae77c4e 2024-12-04T19:16:32,559 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/A/d2efbcb8b45442629764064cba6f4c5b as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/d2efbcb8b45442629764064cba6f4c5b 2024-12-04T19:16:32,561 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/d2efbcb8b45442629764064cba6f4c5b, entries=150, sequenceid=604, filesize=30.5 K 2024-12-04T19:16:32,561 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/B/3e57124039ea413ca506424b8097a237 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/3e57124039ea413ca506424b8097a237 2024-12-04T19:16:32,564 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/3e57124039ea413ca506424b8097a237, entries=150, sequenceid=604, filesize=12.0 K 2024-12-04T19:16:32,564 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/C/bbb108a1f633472093f816f62ae77c4e as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/bbb108a1f633472093f816f62ae77c4e 2024-12-04T19:16:32,566 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/bbb108a1f633472093f816f62ae77c4e, entries=150, sequenceid=604, filesize=12.0 K 2024-12-04T19:16:32,567 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~145.88 KB/149376, currentSize=134.18 KB/137400 for e22a2b2fd059f99265758fd39ea40832 in 839ms, sequenceid=604, compaction requested=true 2024-12-04T19:16:32,567 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e22a2b2fd059f99265758fd39ea40832: 2024-12-04T19:16:32,567 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e22a2b2fd059f99265758fd39ea40832:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T19:16:32,568 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:16:32,568 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T19:16:32,568 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e22a2b2fd059f99265758fd39ea40832:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T19:16:32,568 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:16:32,568 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T19:16:32,568 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e22a2b2fd059f99265758fd39ea40832:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T19:16:32,568 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T19:16:32,568 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49716 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T19:16:32,568 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 134226 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T19:16:32,568 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1541): e22a2b2fd059f99265758fd39ea40832/B is initiating minor compaction (all files) 2024-12-04T19:16:32,568 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1541): e22a2b2fd059f99265758fd39ea40832/A is initiating minor compaction (all files) 2024-12-04T19:16:32,568 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e22a2b2fd059f99265758fd39ea40832/B in TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. 2024-12-04T19:16:32,568 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e22a2b2fd059f99265758fd39ea40832/A in TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. 2024-12-04T19:16:32,568 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/bea39d43bde142b9b4fe87664db4b9d6, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/fdffdd3f750c4cd596210f0a7a958034, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/577dffbcf3924a3db7a3feac0a4361f5, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/3e57124039ea413ca506424b8097a237] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp, totalSize=48.6 K 2024-12-04T19:16:32,568 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/e000a16457964f44aaea140c421f9227, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/12dff66832ba4d97bf39771d440185e9, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/4c7a9594254c4145bb57b14ead897576, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/d2efbcb8b45442629764064cba6f4c5b] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp, totalSize=131.1 K 2024-12-04T19:16:32,568 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. 2024-12-04T19:16:32,569 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. files: [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/e000a16457964f44aaea140c421f9227, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/12dff66832ba4d97bf39771d440185e9, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/4c7a9594254c4145bb57b14ead897576, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/d2efbcb8b45442629764064cba6f4c5b] 2024-12-04T19:16:32,569 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting bea39d43bde142b9b4fe87664db4b9d6, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=528, earliestPutTs=1733339788109 2024-12-04T19:16:32,569 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting e000a16457964f44aaea140c421f9227, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=528, earliestPutTs=1733339788109 2024-12-04T19:16:32,569 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting fdffdd3f750c4cd596210f0a7a958034, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=567, earliestPutTs=1733339788283 2024-12-04T19:16:32,569 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 12dff66832ba4d97bf39771d440185e9, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=567, earliestPutTs=1733339788283 2024-12-04T19:16:32,569 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 577dffbcf3924a3db7a3feac0a4361f5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=579, earliestPutTs=1733339789478 2024-12-04T19:16:32,569 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4c7a9594254c4145bb57b14ead897576, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=579, earliestPutTs=1733339789478 2024-12-04T19:16:32,569 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e57124039ea413ca506424b8097a237, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=604, earliestPutTs=1733339791713 2024-12-04T19:16:32,570 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting d2efbcb8b45442629764064cba6f4c5b, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=604, earliestPutTs=1733339791713 2024-12-04T19:16:32,574 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e22a2b2fd059f99265758fd39ea40832] 2024-12-04T19:16:32,575 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241204888d86bc08fe4903abf586c8569550bb_e22a2b2fd059f99265758fd39ea40832 store=[table=TestAcidGuarantees family=A region=e22a2b2fd059f99265758fd39ea40832] 2024-12-04T19:16:32,575 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e22a2b2fd059f99265758fd39ea40832#B#compaction#515 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:16:32,576 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/B/5e063f67d24f47e0ab95c69b7d1e58e2 is 50, key is test_row_0/B:col10/1733339791713/Put/seqid=0 2024-12-04T19:16:32,577 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241204888d86bc08fe4903abf586c8569550bb_e22a2b2fd059f99265758fd39ea40832, store=[table=TestAcidGuarantees family=A region=e22a2b2fd059f99265758fd39ea40832] 2024-12-04T19:16:32,577 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204888d86bc08fe4903abf586c8569550bb_e22a2b2fd059f99265758fd39ea40832 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e22a2b2fd059f99265758fd39ea40832] 2024-12-04T19:16:32,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742438_1614 (size=12949) 2024-12-04T19:16:32,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742439_1615 (size=4469) 2024-12-04T19:16:32,987 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e22a2b2fd059f99265758fd39ea40832#A#compaction#516 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:16:32,987 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/A/6f887d8ce44f4415aa76158ab2bd5926 is 175, key is test_row_0/A:col10/1733339791713/Put/seqid=0 2024-12-04T19:16:32,988 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/B/5e063f67d24f47e0ab95c69b7d1e58e2 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/5e063f67d24f47e0ab95c69b7d1e58e2 2024-12-04T19:16:32,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742440_1616 (size=31903) 2024-12-04T19:16:32,992 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e22a2b2fd059f99265758fd39ea40832/B of e22a2b2fd059f99265758fd39ea40832 into 5e063f67d24f47e0ab95c69b7d1e58e2(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:16:32,992 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e22a2b2fd059f99265758fd39ea40832: 2024-12-04T19:16:32,993 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., storeName=e22a2b2fd059f99265758fd39ea40832/B, priority=12, startTime=1733339792568; duration=0sec 2024-12-04T19:16:32,993 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T19:16:32,993 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e22a2b2fd059f99265758fd39ea40832:B 2024-12-04T19:16:32,993 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T19:16:32,994 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49716 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T19:16:32,994 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1541): e22a2b2fd059f99265758fd39ea40832/C is initiating minor compaction (all files) 2024-12-04T19:16:32,994 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e22a2b2fd059f99265758fd39ea40832/C in TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. 2024-12-04T19:16:32,994 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/37eaac5afbed447ca4c8e176a7a568ad, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/c53448bd0f554ff8b64dc65c8785b77f, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/d83a9484443546298b44a592d1a0ba97, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/bbb108a1f633472093f816f62ae77c4e] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp, totalSize=48.6 K 2024-12-04T19:16:32,994 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 37eaac5afbed447ca4c8e176a7a568ad, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=528, earliestPutTs=1733339788109 2024-12-04T19:16:32,994 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting c53448bd0f554ff8b64dc65c8785b77f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=567, earliestPutTs=1733339788283 2024-12-04T19:16:32,994 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting d83a9484443546298b44a592d1a0ba97, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=579, earliestPutTs=1733339789478 2024-12-04T19:16:32,995 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting bbb108a1f633472093f816f62ae77c4e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=604, earliestPutTs=1733339791713 2024-12-04T19:16:33,000 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e22a2b2fd059f99265758fd39ea40832#C#compaction#517 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:16:33,000 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/C/4ae44de7a41c4fafba785b679aa151f4 is 50, key is test_row_0/C:col10/1733339791713/Put/seqid=0 2024-12-04T19:16:33,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742441_1617 (size=12949) 2024-12-04T19:16:33,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(8855): Flush requested on e22a2b2fd059f99265758fd39ea40832 2024-12-04T19:16:33,004 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e22a2b2fd059f99265758fd39ea40832 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-12-04T19:16:33,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e22a2b2fd059f99265758fd39ea40832, store=A 2024-12-04T19:16:33,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-04T19:16:33,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e22a2b2fd059f99265758fd39ea40832, store=B 2024-12-04T19:16:33,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-04T19:16:33,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e22a2b2fd059f99265758fd39ea40832, store=C 2024-12-04T19:16:33,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-04T19:16:33,012 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204155d65f8f8f94b92915492ca07a9ce52_e22a2b2fd059f99265758fd39ea40832 is 50, key is test_row_0/A:col10/1733339791846/Put/seqid=0 2024-12-04T19:16:33,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742442_1618 (size=12454) 2024-12-04T19:16:33,033 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:33,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 336 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38946 deadline: 1733339853031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:33,034 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:33,034 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:33,034 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:33,145 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:33,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 337 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38946 deadline: 1733339853142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:33,146 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:33,146 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:33,146 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:33,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-04T19:16:33,192 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T19:16:33,192 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T19:16:33,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-12-04T19:16:33,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-04T19:16:33,193 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T19:16:33,194 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T19:16:33,194 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T19:16:33,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-04T19:16:33,346 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-04T19:16:33,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. 2024-12-04T19:16:33,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. as already flushing 2024-12-04T19:16:33,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. 2024-12-04T19:16:33,346 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:33,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:33,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:33,356 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:33,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 338 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38946 deadline: 1733339853352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:33,356 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:33,356 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:33,357 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:33,394 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/A/6f887d8ce44f4415aa76158ab2bd5926 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/6f887d8ce44f4415aa76158ab2bd5926 2024-12-04T19:16:33,397 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e22a2b2fd059f99265758fd39ea40832/A of e22a2b2fd059f99265758fd39ea40832 into 6f887d8ce44f4415aa76158ab2bd5926(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:16:33,397 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e22a2b2fd059f99265758fd39ea40832: 2024-12-04T19:16:33,397 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., storeName=e22a2b2fd059f99265758fd39ea40832/A, priority=12, startTime=1733339792567; duration=0sec 2024-12-04T19:16:33,397 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:16:33,397 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e22a2b2fd059f99265758fd39ea40832:A 2024-12-04T19:16:33,408 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/C/4ae44de7a41c4fafba785b679aa151f4 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/4ae44de7a41c4fafba785b679aa151f4 2024-12-04T19:16:33,411 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e22a2b2fd059f99265758fd39ea40832/C of e22a2b2fd059f99265758fd39ea40832 into 4ae44de7a41c4fafba785b679aa151f4(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:16:33,411 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e22a2b2fd059f99265758fd39ea40832: 2024-12-04T19:16:33,411 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., storeName=e22a2b2fd059f99265758fd39ea40832/C, priority=12, startTime=1733339792568; duration=0sec 2024-12-04T19:16:33,411 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:16:33,411 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e22a2b2fd059f99265758fd39ea40832:C 2024-12-04T19:16:33,415 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:33,417 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204155d65f8f8f94b92915492ca07a9ce52_e22a2b2fd059f99265758fd39ea40832 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204155d65f8f8f94b92915492ca07a9ce52_e22a2b2fd059f99265758fd39ea40832 2024-12-04T19:16:33,418 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/A/950617064d5e468b939ab90f630c9443, store: [table=TestAcidGuarantees family=A region=e22a2b2fd059f99265758fd39ea40832] 2024-12-04T19:16:33,418 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/A/950617064d5e468b939ab90f630c9443 is 175, key is test_row_0/A:col10/1733339791846/Put/seqid=0 2024-12-04T19:16:33,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742443_1619 (size=31255) 2024-12-04T19:16:33,498 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-04T19:16:33,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. 2024-12-04T19:16:33,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. as already flushing 2024-12-04T19:16:33,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. 2024-12-04T19:16:33,499 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:33,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:33,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:33,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-04T19:16:33,651 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-04T19:16:33,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. 2024-12-04T19:16:33,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. as already flushing 2024-12-04T19:16:33,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. 2024-12-04T19:16:33,651 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:33,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:33,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:33,666 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:33,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 339 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38946 deadline: 1733339853662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:33,666 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:33,666 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:33,666 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:33,803 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-04T19:16:33,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. 2024-12-04T19:16:33,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. as already flushing 2024-12-04T19:16:33,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. 2024-12-04T19:16:33,803 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:33,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:33,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:33,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-04T19:16:33,828 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=643, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/A/950617064d5e468b939ab90f630c9443 2024-12-04T19:16:33,833 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/B/c60683dd934349dfb28d27564e7f8a7e is 50, key is test_row_0/B:col10/1733339791846/Put/seqid=0 2024-12-04T19:16:33,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742444_1620 (size=12301) 2024-12-04T19:16:33,955 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-04T19:16:33,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. 2024-12-04T19:16:33,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. as already flushing 2024-12-04T19:16:33,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. 2024-12-04T19:16:33,956 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:33,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:33,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:34,108 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-04T19:16:34,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. 2024-12-04T19:16:34,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. as already flushing 2024-12-04T19:16:34,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. 2024-12-04T19:16:34,108 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:34,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:34,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:34,174 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:34,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 340 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38946 deadline: 1733339854172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:34,174 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:34,174 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:34,174 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:34,236 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=643 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/B/c60683dd934349dfb28d27564e7f8a7e 2024-12-04T19:16:34,242 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/C/6260e97cfe224c17b8913b944009fd8d is 50, key is test_row_0/C:col10/1733339791846/Put/seqid=0 2024-12-04T19:16:34,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742445_1621 (size=12301) 2024-12-04T19:16:34,260 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-04T19:16:34,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. 2024-12-04T19:16:34,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. as already flushing 2024-12-04T19:16:34,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. 2024-12-04T19:16:34,260 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:34,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:34,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:34,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-04T19:16:34,402 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:34,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38914 deadline: 1733339854402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:34,403 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 9ba2734cd39f,33969,1733339629791 for 'test_row_0' in e22a2b2fd059f99265758fd39ea40832 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-04T19:16:34,403 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:34,404 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:34,404 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:34,412 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-04T19:16:34,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. 2024-12-04T19:16:34,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. as already flushing 2024-12-04T19:16:34,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. 2024-12-04T19:16:34,413 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:34,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:34,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:34,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:34,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 254 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38908 deadline: 1733339854452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:34,454 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 9ba2734cd39f,33969,1733339629791 for 'test_row_2' in e22a2b2fd059f99265758fd39ea40832 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8234 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-04T19:16:34,454 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:34,454 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:34,455 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:34,565 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-04T19:16:34,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. 2024-12-04T19:16:34,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. as already flushing 2024-12-04T19:16:34,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. 2024-12-04T19:16:34,565 ERROR [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:34,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:34,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:16:34,645 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=643 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/C/6260e97cfe224c17b8913b944009fd8d 2024-12-04T19:16:34,648 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/A/950617064d5e468b939ab90f630c9443 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/950617064d5e468b939ab90f630c9443 2024-12-04T19:16:34,657 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/950617064d5e468b939ab90f630c9443, entries=150, sequenceid=643, filesize=30.5 K 2024-12-04T19:16:34,658 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/B/c60683dd934349dfb28d27564e7f8a7e as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/c60683dd934349dfb28d27564e7f8a7e 2024-12-04T19:16:34,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,661 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/c60683dd934349dfb28d27564e7f8a7e, entries=150, sequenceid=643, filesize=12.0 K 2024-12-04T19:16:34,661 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/C/6260e97cfe224c17b8913b944009fd8d as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/6260e97cfe224c17b8913b944009fd8d 2024-12-04T19:16:34,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,663 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/6260e97cfe224c17b8913b944009fd8d, entries=150, sequenceid=643, filesize=12.0 K 2024-12-04T19:16:34,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,664 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~405.56 KB/415296, currentSize=40.25 KB/41220 for e22a2b2fd059f99265758fd39ea40832 in 1660ms, sequenceid=643, compaction requested=false 2024-12-04T19:16:34,664 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e22a2b2fd059f99265758fd39ea40832: 2024-12-04T19:16:34,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,717 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33969 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-04T19:16:34,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. 2024-12-04T19:16:34,718 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing e22a2b2fd059f99265758fd39ea40832 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-12-04T19:16:34,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e22a2b2fd059f99265758fd39ea40832, store=A 2024-12-04T19:16:34,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:16:34,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e22a2b2fd059f99265758fd39ea40832, store=B 2024-12-04T19:16:34,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:16:34,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e22a2b2fd059f99265758fd39ea40832, store=C 2024-12-04T19:16:34,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:16:34,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412045d9a1797c8154f1a880ea1c036af953d_e22a2b2fd059f99265758fd39ea40832 is 50, key is test_row_0/A:col10/1733339793030/Put/seqid=0 2024-12-04T19:16:34,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742446_1622 (size=9914) 2024-12-04T19:16:34,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,737 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412045d9a1797c8154f1a880ea1c036af953d_e22a2b2fd059f99265758fd39ea40832 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412045d9a1797c8154f1a880ea1c036af953d_e22a2b2fd059f99265758fd39ea40832 2024-12-04T19:16:34,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/A/50c1b000c4ab45a9bdf9c98a22eeffbb, store: [table=TestAcidGuarantees family=A region=e22a2b2fd059f99265758fd39ea40832] 2024-12-04T19:16:34,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/A/50c1b000c4ab45a9bdf9c98a22eeffbb is 175, key is test_row_0/A:col10/1733339793030/Put/seqid=0 2024-12-04T19:16:34,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742447_1623 (size=22561) 2024-12-04T19:16:34,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,743 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=654, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/A/50c1b000c4ab45a9bdf9c98a22eeffbb 2024-12-04T19:16:34,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/B/d1b8de816a6f4afca203d8935865b4df is 50, key is test_row_0/B:col10/1733339793030/Put/seqid=0 2024-12-04T19:16:34,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742448_1624 (size=9857) 2024-12-04T19:16:34,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:34,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,165 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=654 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/B/d1b8de816a6f4afca203d8935865b4df 2024-12-04T19:16:35,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/C/7233652bcf1f480f8ff33dec83b46f57 is 50, key is test_row_0/C:col10/1733339793030/Put/seqid=0 2024-12-04T19:16:35,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742449_1625 (size=9857) 2024-12-04T19:16:35,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(8855): Flush requested on e22a2b2fd059f99265758fd39ea40832 2024-12-04T19:16:35,194 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. as already flushing 2024-12-04T19:16:35,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,260 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733339795244, max timestamp=1733339795257Num uniques 859; 2024-12-04T19:16:35,260 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733339795244, max timestamp=1733339795257Num uniques 859; 2024-12-04T19:16:35,260 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T19:16:35,260 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733339795244, max timestamp=1733339795257Num uniques 859; 2024-12-04T19:16:35,260 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T19:16:35,260 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T19:16:35,351 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733339795325, max timestamp=1733339795347Num uniques 1018; 2024-12-04T19:16:35,351 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733339795325, max timestamp=1733339795347Num uniques 1018; 2024-12-04T19:16:35,351 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-04T19:16:35,351 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-04T19:16:35,351 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733339795325, max timestamp=1733339795347Num uniques 1018; 2024-12-04T19:16:35,351 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-04T19:16:35,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-04T19:16:35,429 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733339795408, max timestamp=1733339795426Num uniques 1027; 2024-12-04T19:16:35,429 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-12-04T19:16:35,429 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733339795408, max timestamp=1733339795426Num uniques 1027; 2024-12-04T19:16:35,429 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-12-04T19:16:35,429 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733339795408, max timestamp=1733339795426Num uniques 1027; 2024-12-04T19:16:35,429 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-12-04T19:16:35,497 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733339795439, max timestamp=1733339795494Num uniques 1036; 2024-12-04T19:16:35,497 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-12-04T19:16:35,497 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733339795439, max timestamp=1733339795494Num uniques 1036; 2024-12-04T19:16:35,497 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-12-04T19:16:35,497 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733339795439, max timestamp=1733339795494Num uniques 1036; 2024-12-04T19:16:35,497 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-12-04T19:16:35,564 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1733339795550, max timestamp=1733339795561Num uniques 1045; 2024-12-04T19:16:35,564 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-12-04T19:16:35,564 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1733339795550, max timestamp=1733339795561Num uniques 1045; 2024-12-04T19:16:35,564 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-12-04T19:16:35,564 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1733339795550, max timestamp=1733339795561Num uniques 1045; 2024-12-04T19:16:35,564 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9ba2734cd39f:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-12-04T19:16:35,574 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=654 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/C/7233652bcf1f480f8ff33dec83b46f57 2024-12-04T19:16:35,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/A/50c1b000c4ab45a9bdf9c98a22eeffbb as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/50c1b000c4ab45a9bdf9c98a22eeffbb 2024-12-04T19:16:35,579 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/50c1b000c4ab45a9bdf9c98a22eeffbb, entries=100, sequenceid=654, filesize=22.0 K 2024-12-04T19:16:35,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/B/d1b8de816a6f4afca203d8935865b4df as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/d1b8de816a6f4afca203d8935865b4df 2024-12-04T19:16:35,583 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/d1b8de816a6f4afca203d8935865b4df, entries=100, sequenceid=654, filesize=9.6 K 2024-12-04T19:16:35,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/C/7233652bcf1f480f8ff33dec83b46f57 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/7233652bcf1f480f8ff33dec83b46f57 2024-12-04T19:16:35,586 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/7233652bcf1f480f8ff33dec83b46f57, entries=100, sequenceid=654, filesize=9.6 K 2024-12-04T19:16:35,587 INFO [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=67.09 KB/68700 for e22a2b2fd059f99265758fd39ea40832 in 869ms, sequenceid=654, compaction requested=true 2024-12-04T19:16:35,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for e22a2b2fd059f99265758fd39ea40832: 2024-12-04T19:16:35,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. 2024-12-04T19:16:35,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-12-04T19:16:35,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-12-04T19:16:35,589 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-12-04T19:16:35,589 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3940 sec 2024-12-04T19:16:35,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(8855): Flush requested on e22a2b2fd059f99265758fd39ea40832 2024-12-04T19:16:35,598 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e22a2b2fd059f99265758fd39ea40832 3/3 column families, dataSize=73.80 KB heapSize=202.31 KB 2024-12-04T19:16:35,598 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 2.3980 sec 2024-12-04T19:16:35,598 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e22a2b2fd059f99265758fd39ea40832, store=A 2024-12-04T19:16:35,598 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T19:16:35,598 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e22a2b2fd059f99265758fd39ea40832, store=B 2024-12-04T19:16:35,598 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T19:16:35,598 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e22a2b2fd059f99265758fd39ea40832, store=C 2024-12-04T19:16:35,598 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T19:16:35,603 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204f9b244e57d4b49e4a7a9e2eb1c292689_e22a2b2fd059f99265758fd39ea40832 is 50, key is test_row_0/A:col10/1733339795569/Put/seqid=0 2024-12-04T19:16:35,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742450_1626 (size=14994) 2024-12-04T19:16:35,612 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:35,615 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204f9b244e57d4b49e4a7a9e2eb1c292689_e22a2b2fd059f99265758fd39ea40832 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204f9b244e57d4b49e4a7a9e2eb1c292689_e22a2b2fd059f99265758fd39ea40832 2024-12-04T19:16:35,616 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/A/ea97b7e186674e5fb042b659c7458661, store: [table=TestAcidGuarantees family=A region=e22a2b2fd059f99265758fd39ea40832] 2024-12-04T19:16:35,616 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/A/ea97b7e186674e5fb042b659c7458661 is 175, key is test_row_0/A:col10/1733339795569/Put/seqid=0 2024-12-04T19:16:35,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742451_1627 (size=39949) 2024-12-04T19:16:35,665 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:35,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 450 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38946 deadline: 1733339855662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:35,665 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:35,665 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:35,665 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:35,773 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:35,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 451 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38946 deadline: 1733339855772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:35,773 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:35,773 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:35,773 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:35,981 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:35,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 452 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38946 deadline: 1733339855981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:35,982 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:35,982 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:35,982 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:36,019 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=750, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/A/ea97b7e186674e5fb042b659c7458661 2024-12-04T19:16:36,024 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/B/bd928be9c4a344968b130ccc125dfb23 is 50, key is test_row_0/B:col10/1733339795569/Put/seqid=0 2024-12-04T19:16:36,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742452_1628 (size=12301) 2024-12-04T19:16:36,295 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:36,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 453 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38946 deadline: 1733339856291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:36,295 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:36,295 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:36,295 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:36,428 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=750 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/B/bd928be9c4a344968b130ccc125dfb23 2024-12-04T19:16:36,432 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/C/f28cc6129af8492683331ffd31d4db7c is 50, key is test_row_0/C:col10/1733339795569/Put/seqid=0 2024-12-04T19:16:36,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742453_1629 (size=12301) 2024-12-04T19:16:36,803 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:36,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38922 deadline: 1733339856802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:36,803 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:36,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 454 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38946 deadline: 1733339856802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:36,804 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:36,804 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:36,804 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:36,803 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 9ba2734cd39f,33969,1733339629791 for 'test_row_0' in e22a2b2fd059f99265758fd39ea40832 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18260 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-04T19:16:36,804 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:36,804 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:36,804 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:36,834 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:16:36,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33969 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38904 deadline: 1733339856832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:36,835 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 9ba2734cd39f,33969,1733339629791 for 'test_row_2' in e22a2b2fd059f99265758fd39ea40832 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18288 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-04T19:16:36,835 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 , the old value is region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:36,835 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T19:16:36,835 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., hostname=9ba2734cd39f,33969,1733339629791, seqNum=5 because the exception is null or not the one we care about 2024-12-04T19:16:36,836 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=750 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/C/f28cc6129af8492683331ffd31d4db7c 2024-12-04T19:16:36,839 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/A/ea97b7e186674e5fb042b659c7458661 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/ea97b7e186674e5fb042b659c7458661 2024-12-04T19:16:36,842 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/ea97b7e186674e5fb042b659c7458661, entries=200, sequenceid=750, filesize=39.0 K 2024-12-04T19:16:36,843 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/B/bd928be9c4a344968b130ccc125dfb23 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/bd928be9c4a344968b130ccc125dfb23 2024-12-04T19:16:36,845 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/bd928be9c4a344968b130ccc125dfb23, entries=150, sequenceid=750, filesize=12.0 K 2024-12-04T19:16:36,846 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/C/f28cc6129af8492683331ffd31d4db7c as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/f28cc6129af8492683331ffd31d4db7c 2024-12-04T19:16:36,848 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/f28cc6129af8492683331ffd31d4db7c, entries=150, sequenceid=750, filesize=12.0 K 2024-12-04T19:16:36,849 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~202.27 KB/207120, currentSize=107.34 KB/109920 for e22a2b2fd059f99265758fd39ea40832 in 1252ms, sequenceid=750, compaction requested=true 2024-12-04T19:16:36,849 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e22a2b2fd059f99265758fd39ea40832: 2024-12-04T19:16:36,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e22a2b2fd059f99265758fd39ea40832:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T19:16:36,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:16:36,849 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T19:16:36,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e22a2b2fd059f99265758fd39ea40832:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T19:16:36,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:16:36,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e22a2b2fd059f99265758fd39ea40832:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T19:16:36,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T19:16:36,849 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T19:16:36,850 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47408 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T19:16:36,850 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 125668 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T19:16:36,850 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1541): e22a2b2fd059f99265758fd39ea40832/B is initiating minor compaction (all files) 2024-12-04T19:16:36,850 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1541): e22a2b2fd059f99265758fd39ea40832/A is initiating minor compaction (all files) 2024-12-04T19:16:36,850 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e22a2b2fd059f99265758fd39ea40832/A in TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. 2024-12-04T19:16:36,850 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e22a2b2fd059f99265758fd39ea40832/B in TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. 2024-12-04T19:16:36,850 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/6f887d8ce44f4415aa76158ab2bd5926, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/950617064d5e468b939ab90f630c9443, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/50c1b000c4ab45a9bdf9c98a22eeffbb, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/ea97b7e186674e5fb042b659c7458661] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp, totalSize=122.7 K 2024-12-04T19:16:36,850 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/5e063f67d24f47e0ab95c69b7d1e58e2, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/c60683dd934349dfb28d27564e7f8a7e, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/d1b8de816a6f4afca203d8935865b4df, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/bd928be9c4a344968b130ccc125dfb23] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp, totalSize=46.3 K 2024-12-04T19:16:36,850 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. 2024-12-04T19:16:36,850 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. files: [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/6f887d8ce44f4415aa76158ab2bd5926, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/950617064d5e468b939ab90f630c9443, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/50c1b000c4ab45a9bdf9c98a22eeffbb, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/ea97b7e186674e5fb042b659c7458661] 2024-12-04T19:16:36,851 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e063f67d24f47e0ab95c69b7d1e58e2, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=604, earliestPutTs=1733339791713 2024-12-04T19:16:36,851 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6f887d8ce44f4415aa76158ab2bd5926, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=604, earliestPutTs=1733339791713 2024-12-04T19:16:36,851 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 950617064d5e468b939ab90f630c9443, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=643, earliestPutTs=1733339791832 2024-12-04T19:16:36,851 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting c60683dd934349dfb28d27564e7f8a7e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=643, earliestPutTs=1733339791832 2024-12-04T19:16:36,851 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting d1b8de816a6f4afca203d8935865b4df, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=654, earliestPutTs=1733339793026 2024-12-04T19:16:36,852 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 50c1b000c4ab45a9bdf9c98a22eeffbb, keycount=100, bloomtype=ROW, size=22.0 K, encoding=NONE, compression=NONE, seqNum=654, earliestPutTs=1733339793026 2024-12-04T19:16:36,852 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] compactions.Compactor(225): Compacting bd928be9c4a344968b130ccc125dfb23, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=750, earliestPutTs=1733339795569 2024-12-04T19:16:36,852 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting ea97b7e186674e5fb042b659c7458661, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=750, earliestPutTs=1733339795569 2024-12-04T19:16:36,856 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e22a2b2fd059f99265758fd39ea40832] 2024-12-04T19:16:36,857 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e22a2b2fd059f99265758fd39ea40832#B#compaction#528 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:16:36,857 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241204fa2068826409497bb314dc2873232945_e22a2b2fd059f99265758fd39ea40832 store=[table=TestAcidGuarantees family=A region=e22a2b2fd059f99265758fd39ea40832] 2024-12-04T19:16:36,857 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/B/b19ba86cdff54b8bb05ff38cd8bf6cc9 is 50, key is test_row_0/B:col10/1733339795569/Put/seqid=0 2024-12-04T19:16:36,859 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241204fa2068826409497bb314dc2873232945_e22a2b2fd059f99265758fd39ea40832, store=[table=TestAcidGuarantees family=A region=e22a2b2fd059f99265758fd39ea40832] 2024-12-04T19:16:36,859 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204fa2068826409497bb314dc2873232945_e22a2b2fd059f99265758fd39ea40832 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e22a2b2fd059f99265758fd39ea40832] 2024-12-04T19:16:36,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742454_1630 (size=13085) 2024-12-04T19:16:36,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742455_1631 (size=4469) 2024-12-04T19:16:36,863 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e22a2b2fd059f99265758fd39ea40832#A#compaction#527 average throughput is 4.07 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:16:36,863 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/A/2593637b2b9849e8bb453b0391820449 is 175, key is test_row_0/A:col10/1733339795569/Put/seqid=0 2024-12-04T19:16:36,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742456_1632 (size=32039) 2024-12-04T19:16:36,869 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/A/2593637b2b9849e8bb453b0391820449 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/2593637b2b9849e8bb453b0391820449 2024-12-04T19:16:36,872 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e22a2b2fd059f99265758fd39ea40832/A of e22a2b2fd059f99265758fd39ea40832 into 2593637b2b9849e8bb453b0391820449(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:16:36,872 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e22a2b2fd059f99265758fd39ea40832: 2024-12-04T19:16:36,872 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., storeName=e22a2b2fd059f99265758fd39ea40832/A, priority=12, startTime=1733339796849; duration=0sec 2024-12-04T19:16:36,872 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T19:16:36,872 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e22a2b2fd059f99265758fd39ea40832:A 2024-12-04T19:16:36,872 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T19:16:36,874 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47408 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T19:16:36,874 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1541): e22a2b2fd059f99265758fd39ea40832/C is initiating minor compaction (all files) 2024-12-04T19:16:36,874 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e22a2b2fd059f99265758fd39ea40832/C in TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. 2024-12-04T19:16:36,874 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/4ae44de7a41c4fafba785b679aa151f4, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/6260e97cfe224c17b8913b944009fd8d, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/7233652bcf1f480f8ff33dec83b46f57, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/f28cc6129af8492683331ffd31d4db7c] into tmpdir=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp, totalSize=46.3 K 2024-12-04T19:16:36,874 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ae44de7a41c4fafba785b679aa151f4, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=604, earliestPutTs=1733339791713 2024-12-04T19:16:36,874 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6260e97cfe224c17b8913b944009fd8d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=643, earliestPutTs=1733339791832 2024-12-04T19:16:36,875 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7233652bcf1f480f8ff33dec83b46f57, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=654, earliestPutTs=1733339793026 2024-12-04T19:16:36,875 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] compactions.Compactor(225): Compacting f28cc6129af8492683331ffd31d4db7c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=750, earliestPutTs=1733339795569 2024-12-04T19:16:36,880 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e22a2b2fd059f99265758fd39ea40832#C#compaction#529 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:16:36,880 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/C/a1cdeb3e281242d38919968262bc67e1 is 50, key is test_row_0/C:col10/1733339795569/Put/seqid=0 2024-12-04T19:16:36,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742457_1633 (size=13085) 2024-12-04T19:16:37,256 INFO [Thread-2406 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2406. 2024-12-04T19:16:37,256 DEBUG [Thread-2406 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T19:16:37,256 DEBUG [Thread-2406 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:16:37,256 DEBUG [Thread-2406 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:16:37,256 INFO [Thread-2408 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2408. 2024-12-04T19:16:37,256 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T19:16:37,257 DEBUG [Thread-2408 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T19:16:37,257 DEBUG [Thread-2408 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:16:37,257 DEBUG [Thread-2408 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:16:37,257 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T19:16:37,259 INFO [Thread-2402 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2402. 2024-12-04T19:16:37,259 DEBUG [Thread-2402 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T19:16:37,259 DEBUG [Thread-2402 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:16:37,259 DEBUG [Thread-2402 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:16:37,259 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T19:16:37,259 INFO [Thread-2410 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2410. 2024-12-04T19:16:37,259 DEBUG [Thread-2410 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T19:16:37,259 DEBUG [Thread-2410 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:16:37,259 DEBUG [Thread-2410 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:16:37,259 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T19:16:37,259 INFO [Thread-2404 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2404. 2024-12-04T19:16:37,259 DEBUG [Thread-2404 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T19:16:37,259 DEBUG [Thread-2404 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:16:37,259 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T19:16:37,259 DEBUG [Thread-2404 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:16:37,264 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/B/b19ba86cdff54b8bb05ff38cd8bf6cc9 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/b19ba86cdff54b8bb05ff38cd8bf6cc9 2024-12-04T19:16:37,267 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e22a2b2fd059f99265758fd39ea40832/B of e22a2b2fd059f99265758fd39ea40832 into b19ba86cdff54b8bb05ff38cd8bf6cc9(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:16:37,267 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e22a2b2fd059f99265758fd39ea40832: 2024-12-04T19:16:37,267 INFO [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., storeName=e22a2b2fd059f99265758fd39ea40832/B, priority=12, startTime=1733339796849; duration=0sec 2024-12-04T19:16:37,267 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:16:37,267 DEBUG [RS:0;9ba2734cd39f:33969-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e22a2b2fd059f99265758fd39ea40832:B 2024-12-04T19:16:37,286 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/C/a1cdeb3e281242d38919968262bc67e1 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/a1cdeb3e281242d38919968262bc67e1 2024-12-04T19:16:37,289 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e22a2b2fd059f99265758fd39ea40832/C of e22a2b2fd059f99265758fd39ea40832 into a1cdeb3e281242d38919968262bc67e1(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:16:37,289 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e22a2b2fd059f99265758fd39ea40832: 2024-12-04T19:16:37,289 INFO [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., storeName=e22a2b2fd059f99265758fd39ea40832/C, priority=12, startTime=1733339796849; duration=0sec 2024-12-04T19:16:37,289 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T19:16:37,289 DEBUG [RS:0;9ba2734cd39f:33969-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e22a2b2fd059f99265758fd39ea40832:C 2024-12-04T19:16:37,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-04T19:16:37,382 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T19:16:37,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33969 {}] regionserver.HRegion(8855): Flush requested on e22a2b2fd059f99265758fd39ea40832 2024-12-04T19:16:37,816 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e22a2b2fd059f99265758fd39ea40832 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-04T19:16:37,817 INFO [Thread-2399 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2399. 2024-12-04T19:16:37,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e22a2b2fd059f99265758fd39ea40832, store=A 2024-12-04T19:16:37,817 DEBUG [Thread-2399 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T19:16:37,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:16:37,817 DEBUG [Thread-2399 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:16:37,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e22a2b2fd059f99265758fd39ea40832, store=B 2024-12-04T19:16:37,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:16:37,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e22a2b2fd059f99265758fd39ea40832, store=C 2024-12-04T19:16:37,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:16:37,817 DEBUG [Thread-2399 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:16:37,817 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T19:16:37,826 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412048745c6591dbc4110b951d901ddf736b2_e22a2b2fd059f99265758fd39ea40832 is 50, key is test_row_0/A:col10/1733339797813/Put/seqid=0 2024-12-04T19:16:37,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742458_1634 (size=12454) 2024-12-04T19:16:38,230 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:38,234 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412048745c6591dbc4110b951d901ddf736b2_e22a2b2fd059f99265758fd39ea40832 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412048745c6591dbc4110b951d901ddf736b2_e22a2b2fd059f99265758fd39ea40832 2024-12-04T19:16:38,234 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/A/b066c092d0004dd59b5a12c4e8f4f702, store: [table=TestAcidGuarantees family=A region=e22a2b2fd059f99265758fd39ea40832] 2024-12-04T19:16:38,235 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/A/b066c092d0004dd59b5a12c4e8f4f702 is 175, key is test_row_0/A:col10/1733339797813/Put/seqid=0 2024-12-04T19:16:38,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742459_1635 (size=31255) 2024-12-04T19:16:38,639 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=773, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/A/b066c092d0004dd59b5a12c4e8f4f702 2024-12-04T19:16:38,644 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/B/a5eaed1ac6d8445f9e22b8589cdc83fe is 50, key is test_row_0/B:col10/1733339797813/Put/seqid=0 2024-12-04T19:16:38,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742460_1636 (size=12301) 2024-12-04T19:16:39,047 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=773 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/B/a5eaed1ac6d8445f9e22b8589cdc83fe 2024-12-04T19:16:39,052 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/C/2254c3e1309e4d2fa23a96583d86d2cb is 50, key is test_row_0/C:col10/1733339797813/Put/seqid=0 2024-12-04T19:16:39,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742461_1637 (size=12301) 2024-12-04T19:16:39,456 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=773 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/C/2254c3e1309e4d2fa23a96583d86d2cb 2024-12-04T19:16:39,459 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/A/b066c092d0004dd59b5a12c4e8f4f702 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/b066c092d0004dd59b5a12c4e8f4f702 2024-12-04T19:16:39,462 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/b066c092d0004dd59b5a12c4e8f4f702, entries=150, sequenceid=773, filesize=30.5 K 2024-12-04T19:16:39,463 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/B/a5eaed1ac6d8445f9e22b8589cdc83fe as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/a5eaed1ac6d8445f9e22b8589cdc83fe 2024-12-04T19:16:39,465 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/a5eaed1ac6d8445f9e22b8589cdc83fe, entries=150, sequenceid=773, filesize=12.0 K 2024-12-04T19:16:39,466 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/C/2254c3e1309e4d2fa23a96583d86d2cb as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/2254c3e1309e4d2fa23a96583d86d2cb 2024-12-04T19:16:39,468 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/2254c3e1309e4d2fa23a96583d86d2cb, entries=150, sequenceid=773, filesize=12.0 K 2024-12-04T19:16:39,468 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=0 B/0 for e22a2b2fd059f99265758fd39ea40832 in 1652ms, sequenceid=773, compaction requested=false 2024-12-04T19:16:39,469 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e22a2b2fd059f99265758fd39ea40832: 2024-12-04T19:16:44,453 INFO [Thread-2395 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2395. 2024-12-04T19:16:44,453 DEBUG [Thread-2395 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T19:16:44,453 DEBUG [Thread-2395 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:16:44,453 DEBUG [Thread-2395 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:16:44,453 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T19:16:44,493 INFO [Thread-2393 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2393. 2024-12-04T19:16:44,493 DEBUG [Thread-2393 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T19:16:44,493 DEBUG [Thread-2393 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:16:44,493 DEBUG [Thread-2393 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:16:44,493 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T19:16:46,833 INFO [Thread-2391 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2391. 2024-12-04T19:16:46,833 DEBUG [Thread-2391 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T19:16:46,833 DEBUG [Thread-2391 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:16:46,833 DEBUG [Thread-2391 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:16:46,833 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T19:16:46,853 INFO [Thread-2397 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2397. 2024-12-04T19:16:46,853 DEBUG [Thread-2397 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T19:16:46,853 DEBUG [Thread-2397 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:16:46,853 DEBUG [Thread-2397 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:16:46,853 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T19:16:46,854 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-04T19:16:46,854 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 12 2024-12-04T19:16:46,854 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 222 2024-12-04T19:16:46,854 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 39 2024-12-04T19:16:46,854 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 17 2024-12-04T19:16:46,854 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 408 2024-12-04T19:16:46,854 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-04T19:16:46,854 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3809 2024-12-04T19:16:46,854 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3876 2024-12-04T19:16:46,854 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3693 2024-12-04T19:16:46,854 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3804 2024-12-04T19:16:46,854 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3857 2024-12-04T19:16:46,854 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-04T19:16:46,854 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-04T19:16:46,854 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T19:16:46,854 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:16:46,854 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:16:46,854 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T19:16:46,855 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-04T19:16:46,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-04T19:16:46,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-04T19:16:46,857 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733339806857"}]},"ts":"1733339806857"} 2024-12-04T19:16:46,858 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-04T19:16:46,858 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-04T19:16:46,859 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-04T19:16:46,860 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=164, ppid=163, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e22a2b2fd059f99265758fd39ea40832, UNASSIGN}] 2024-12-04T19:16:46,861 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=164, ppid=163, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e22a2b2fd059f99265758fd39ea40832, UNASSIGN 2024-12-04T19:16:46,861 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=164 updating hbase:meta row=e22a2b2fd059f99265758fd39ea40832, regionState=CLOSING, regionLocation=9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:46,862 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=164, ppid=163, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e22a2b2fd059f99265758fd39ea40832, UNASSIGN because future has completed 2024-12-04T19:16:46,863 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-04T19:16:46,863 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; CloseRegionProcedure e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791}] 2024-12-04T19:16:46,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-04T19:16:47,015 INFO [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] handler.UnassignRegionHandler(122): Close e22a2b2fd059f99265758fd39ea40832 2024-12-04T19:16:47,015 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-04T19:16:47,015 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1722): Closing e22a2b2fd059f99265758fd39ea40832, disabling compactions & flushes 2024-12-04T19:16:47,015 INFO [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. 2024-12-04T19:16:47,015 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. 2024-12-04T19:16:47,015 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. after waiting 0 ms 2024-12-04T19:16:47,015 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. 2024-12-04T19:16:47,015 INFO [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(2902): Flushing e22a2b2fd059f99265758fd39ea40832 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-12-04T19:16:47,015 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e22a2b2fd059f99265758fd39ea40832, store=A 2024-12-04T19:16:47,015 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:16:47,015 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e22a2b2fd059f99265758fd39ea40832, store=B 2024-12-04T19:16:47,016 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:16:47,016 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e22a2b2fd059f99265758fd39ea40832, store=C 2024-12-04T19:16:47,016 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T19:16:47,020 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120456c8e403981f4a379eb5c686114b85b2_e22a2b2fd059f99265758fd39ea40832 is 50, key is test_row_0/A:col10/1733339806832/Put/seqid=0 2024-12-04T19:16:47,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742462_1638 (size=9914) 2024-12-04T19:16:47,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-04T19:16:47,426 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:16:47,432 INFO [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120456c8e403981f4a379eb5c686114b85b2_e22a2b2fd059f99265758fd39ea40832 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120456c8e403981f4a379eb5c686114b85b2_e22a2b2fd059f99265758fd39ea40832 2024-12-04T19:16:47,433 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/A/58a47ba3d7ac4a15bfd9f14ff6a12fa9, store: [table=TestAcidGuarantees family=A region=e22a2b2fd059f99265758fd39ea40832] 2024-12-04T19:16:47,434 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/A/58a47ba3d7ac4a15bfd9f14ff6a12fa9 is 175, key is test_row_0/A:col10/1733339806832/Put/seqid=0 2024-12-04T19:16:47,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742463_1639 (size=22561) 2024-12-04T19:16:47,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-04T19:16:47,840 INFO [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=780, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/A/58a47ba3d7ac4a15bfd9f14ff6a12fa9 2024-12-04T19:16:47,852 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/B/feb36dca1753400ca82ce378f4bae4ff is 50, key is test_row_0/B:col10/1733339806832/Put/seqid=0 2024-12-04T19:16:47,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742464_1640 (size=9857) 2024-12-04T19:16:47,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-04T19:16:48,015 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-04T19:16:48,258 INFO [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=780 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/B/feb36dca1753400ca82ce378f4bae4ff 2024-12-04T19:16:48,272 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/C/3420eaf30ea74b9c8dc6cc92442c6aab is 50, key is test_row_0/C:col10/1733339806832/Put/seqid=0 2024-12-04T19:16:48,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742465_1641 (size=9857) 2024-12-04T19:16:48,677 INFO [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=780 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/C/3420eaf30ea74b9c8dc6cc92442c6aab 2024-12-04T19:16:48,688 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/A/58a47ba3d7ac4a15bfd9f14ff6a12fa9 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/58a47ba3d7ac4a15bfd9f14ff6a12fa9 2024-12-04T19:16:48,694 INFO [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/58a47ba3d7ac4a15bfd9f14ff6a12fa9, entries=100, sequenceid=780, filesize=22.0 K 2024-12-04T19:16:48,694 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/B/feb36dca1753400ca82ce378f4bae4ff as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/feb36dca1753400ca82ce378f4bae4ff 2024-12-04T19:16:48,698 INFO [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/feb36dca1753400ca82ce378f4bae4ff, entries=100, sequenceid=780, filesize=9.6 K 2024-12-04T19:16:48,698 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/.tmp/C/3420eaf30ea74b9c8dc6cc92442c6aab as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/3420eaf30ea74b9c8dc6cc92442c6aab 2024-12-04T19:16:48,702 INFO [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/3420eaf30ea74b9c8dc6cc92442c6aab, entries=100, sequenceid=780, filesize=9.6 K 2024-12-04T19:16:48,702 INFO [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for e22a2b2fd059f99265758fd39ea40832 in 1687ms, sequenceid=780, compaction requested=true 2024-12-04T19:16:48,703 DEBUG [StoreCloser-TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/e9840326039a48a186a612cb083952c7, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/6e8ada5fc5614ae6a4dea92a5a8ddd99, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/f2964597d875459886ebf4e53112c0fa, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/cf2502f3a22c492db4dffecb5a394943, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/a72a42035aef486594852110120ee0a2, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/7da7e18914a445289bf8345906b1e65f, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/57159d1437b94fd2bcfa60f09c0102c2, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/fb29d6266a644dec9c81e077a70bccec, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/9f48a10803ae45208609adcf876e1759, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/a2b6fc038d424b628b70340cbe9f34ed, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/36eeafb63b684336ba016d86ab4f66af, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/f53eb536d0334201b3de728485a691a7, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/9511eacd509244ad8b58d5eefc0f60fe, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/d3619cac95174c1ca6ffe45027cb97e9, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/2761006941da4859bb159b5301ba594e, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/e000a16457964f44aaea140c421f9227, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/12dff66832ba4d97bf39771d440185e9, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/4c7a9594254c4145bb57b14ead897576, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/6f887d8ce44f4415aa76158ab2bd5926, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/d2efbcb8b45442629764064cba6f4c5b, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/950617064d5e468b939ab90f630c9443, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/50c1b000c4ab45a9bdf9c98a22eeffbb, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/ea97b7e186674e5fb042b659c7458661] to archive 2024-12-04T19:16:48,704 DEBUG [StoreCloser-TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-04T19:16:48,705 DEBUG [StoreCloser-TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/e9840326039a48a186a612cb083952c7 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/e9840326039a48a186a612cb083952c7 2024-12-04T19:16:48,707 DEBUG [StoreCloser-TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/6e8ada5fc5614ae6a4dea92a5a8ddd99 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/6e8ada5fc5614ae6a4dea92a5a8ddd99 2024-12-04T19:16:48,708 DEBUG [StoreCloser-TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/f2964597d875459886ebf4e53112c0fa to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/f2964597d875459886ebf4e53112c0fa 2024-12-04T19:16:48,710 DEBUG [StoreCloser-TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/cf2502f3a22c492db4dffecb5a394943 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/cf2502f3a22c492db4dffecb5a394943 2024-12-04T19:16:48,711 DEBUG [StoreCloser-TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/a72a42035aef486594852110120ee0a2 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/a72a42035aef486594852110120ee0a2 2024-12-04T19:16:48,712 DEBUG [StoreCloser-TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/7da7e18914a445289bf8345906b1e65f to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/7da7e18914a445289bf8345906b1e65f 2024-12-04T19:16:48,714 DEBUG [StoreCloser-TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/57159d1437b94fd2bcfa60f09c0102c2 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/57159d1437b94fd2bcfa60f09c0102c2 2024-12-04T19:16:48,715 DEBUG [StoreCloser-TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/fb29d6266a644dec9c81e077a70bccec to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/fb29d6266a644dec9c81e077a70bccec 2024-12-04T19:16:48,717 DEBUG [StoreCloser-TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/9f48a10803ae45208609adcf876e1759 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/9f48a10803ae45208609adcf876e1759 2024-12-04T19:16:48,718 DEBUG [StoreCloser-TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/a2b6fc038d424b628b70340cbe9f34ed to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/a2b6fc038d424b628b70340cbe9f34ed 2024-12-04T19:16:48,720 DEBUG [StoreCloser-TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/36eeafb63b684336ba016d86ab4f66af to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/36eeafb63b684336ba016d86ab4f66af 2024-12-04T19:16:48,722 DEBUG [StoreCloser-TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/f53eb536d0334201b3de728485a691a7 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/f53eb536d0334201b3de728485a691a7 2024-12-04T19:16:48,723 DEBUG [StoreCloser-TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/9511eacd509244ad8b58d5eefc0f60fe to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/9511eacd509244ad8b58d5eefc0f60fe 2024-12-04T19:16:48,725 DEBUG [StoreCloser-TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/d3619cac95174c1ca6ffe45027cb97e9 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/d3619cac95174c1ca6ffe45027cb97e9 2024-12-04T19:16:48,726 DEBUG [StoreCloser-TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/2761006941da4859bb159b5301ba594e to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/2761006941da4859bb159b5301ba594e 2024-12-04T19:16:48,727 DEBUG [StoreCloser-TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/e000a16457964f44aaea140c421f9227 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/e000a16457964f44aaea140c421f9227 2024-12-04T19:16:48,727 DEBUG [StoreCloser-TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/12dff66832ba4d97bf39771d440185e9 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/12dff66832ba4d97bf39771d440185e9 2024-12-04T19:16:48,728 DEBUG [StoreCloser-TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/4c7a9594254c4145bb57b14ead897576 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/4c7a9594254c4145bb57b14ead897576 2024-12-04T19:16:48,729 DEBUG [StoreCloser-TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/6f887d8ce44f4415aa76158ab2bd5926 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/6f887d8ce44f4415aa76158ab2bd5926 2024-12-04T19:16:48,730 DEBUG [StoreCloser-TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/d2efbcb8b45442629764064cba6f4c5b to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/d2efbcb8b45442629764064cba6f4c5b 2024-12-04T19:16:48,731 DEBUG [StoreCloser-TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/950617064d5e468b939ab90f630c9443 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/950617064d5e468b939ab90f630c9443 2024-12-04T19:16:48,732 DEBUG [StoreCloser-TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/50c1b000c4ab45a9bdf9c98a22eeffbb to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/50c1b000c4ab45a9bdf9c98a22eeffbb 2024-12-04T19:16:48,733 DEBUG [StoreCloser-TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/ea97b7e186674e5fb042b659c7458661 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/ea97b7e186674e5fb042b659c7458661 2024-12-04T19:16:48,734 DEBUG [StoreCloser-TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/fc097f27a101452eba3408fa6e892e3d, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/960c8e993e7b47bea1e536660db46177, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/6a48d09d5eab4be997bf96850b7c637e, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/639b99abe7c64a479e7d2d343d01dc10, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/3997e2a8756e43b3ab90f0f40d46e8f4, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/3c49a0fb6b1a42c1aa09c7126d22582c, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/99b16982d0a4470bb478a045ef63b33f, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/64018f244a93466aa3eb37e222b79cda, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/785e4c4c9f7e4d91975486886083afae, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/ecdd77d1d2374e438bd9b2dd6b202a8d, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/3582ef5bd1e1469f9305b6346d7de84b, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/0c7ee0ee94d4462383e9187079fb85a6, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/73761bfc71274eba974347395a06c1d7, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/e795285d9ba348d4ad6c9ffc5590c8ab, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/bea39d43bde142b9b4fe87664db4b9d6, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/d6d0599d7c3c470788ace562699b81ee, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/fdffdd3f750c4cd596210f0a7a958034, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/577dffbcf3924a3db7a3feac0a4361f5, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/5e063f67d24f47e0ab95c69b7d1e58e2, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/3e57124039ea413ca506424b8097a237, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/c60683dd934349dfb28d27564e7f8a7e, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/d1b8de816a6f4afca203d8935865b4df, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/bd928be9c4a344968b130ccc125dfb23] to archive 2024-12-04T19:16:48,735 DEBUG [StoreCloser-TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-04T19:16:48,737 DEBUG [StoreCloser-TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/fc097f27a101452eba3408fa6e892e3d to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/fc097f27a101452eba3408fa6e892e3d 2024-12-04T19:16:48,738 DEBUG [StoreCloser-TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/960c8e993e7b47bea1e536660db46177 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/960c8e993e7b47bea1e536660db46177 2024-12-04T19:16:48,739 DEBUG [StoreCloser-TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/6a48d09d5eab4be997bf96850b7c637e to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/6a48d09d5eab4be997bf96850b7c637e 2024-12-04T19:16:48,741 DEBUG [StoreCloser-TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/639b99abe7c64a479e7d2d343d01dc10 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/639b99abe7c64a479e7d2d343d01dc10 2024-12-04T19:16:48,742 DEBUG [StoreCloser-TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/3997e2a8756e43b3ab90f0f40d46e8f4 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/3997e2a8756e43b3ab90f0f40d46e8f4 2024-12-04T19:16:48,743 DEBUG [StoreCloser-TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/3c49a0fb6b1a42c1aa09c7126d22582c to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/3c49a0fb6b1a42c1aa09c7126d22582c 2024-12-04T19:16:48,745 DEBUG [StoreCloser-TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/99b16982d0a4470bb478a045ef63b33f to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/99b16982d0a4470bb478a045ef63b33f 2024-12-04T19:16:48,746 DEBUG [StoreCloser-TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/64018f244a93466aa3eb37e222b79cda to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/64018f244a93466aa3eb37e222b79cda 2024-12-04T19:16:48,747 DEBUG [StoreCloser-TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/785e4c4c9f7e4d91975486886083afae to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/785e4c4c9f7e4d91975486886083afae 2024-12-04T19:16:48,748 DEBUG [StoreCloser-TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/ecdd77d1d2374e438bd9b2dd6b202a8d to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/ecdd77d1d2374e438bd9b2dd6b202a8d 2024-12-04T19:16:48,749 DEBUG [StoreCloser-TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/3582ef5bd1e1469f9305b6346d7de84b to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/3582ef5bd1e1469f9305b6346d7de84b 2024-12-04T19:16:48,750 DEBUG [StoreCloser-TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/0c7ee0ee94d4462383e9187079fb85a6 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/0c7ee0ee94d4462383e9187079fb85a6 2024-12-04T19:16:48,751 DEBUG [StoreCloser-TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/73761bfc71274eba974347395a06c1d7 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/73761bfc71274eba974347395a06c1d7 2024-12-04T19:16:48,752 DEBUG [StoreCloser-TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/e795285d9ba348d4ad6c9ffc5590c8ab to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/e795285d9ba348d4ad6c9ffc5590c8ab 2024-12-04T19:16:48,753 DEBUG [StoreCloser-TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/bea39d43bde142b9b4fe87664db4b9d6 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/bea39d43bde142b9b4fe87664db4b9d6 2024-12-04T19:16:48,754 DEBUG [StoreCloser-TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/d6d0599d7c3c470788ace562699b81ee to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/d6d0599d7c3c470788ace562699b81ee 2024-12-04T19:16:48,755 DEBUG [StoreCloser-TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/fdffdd3f750c4cd596210f0a7a958034 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/fdffdd3f750c4cd596210f0a7a958034 2024-12-04T19:16:48,756 DEBUG [StoreCloser-TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/577dffbcf3924a3db7a3feac0a4361f5 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/577dffbcf3924a3db7a3feac0a4361f5 2024-12-04T19:16:48,757 DEBUG [StoreCloser-TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/5e063f67d24f47e0ab95c69b7d1e58e2 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/5e063f67d24f47e0ab95c69b7d1e58e2 2024-12-04T19:16:48,757 DEBUG [StoreCloser-TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/3e57124039ea413ca506424b8097a237 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/3e57124039ea413ca506424b8097a237 2024-12-04T19:16:48,758 DEBUG [StoreCloser-TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/c60683dd934349dfb28d27564e7f8a7e to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/c60683dd934349dfb28d27564e7f8a7e 2024-12-04T19:16:48,759 DEBUG [StoreCloser-TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/d1b8de816a6f4afca203d8935865b4df to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/d1b8de816a6f4afca203d8935865b4df 2024-12-04T19:16:48,759 DEBUG [StoreCloser-TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/bd928be9c4a344968b130ccc125dfb23 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/bd928be9c4a344968b130ccc125dfb23 2024-12-04T19:16:48,760 DEBUG [StoreCloser-TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/79ccd2ff28b54a96842ea98b8af18281, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/bd2202b4a0a54318a5b6cd829d6a1118, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/0127ace727de4eeca25fc4c7b6984a09, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/6c24293234724f478ca10a56c33300d4, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/01b887b3f0cd49d49bb361c4e67c6fc8, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/c06df5860725447fbac6bcef54a9402c, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/7c1857b1f5d544d0a6a56699ea2b9e6a, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/e284e87bc2f643d9a1018db9a18f4560, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/5aaea413efe6478e9485ac530987a91f, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/becd0bfa340543349ddc8f2e15c8f77d, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/8f7d3df6a4ae48c8a9e97619f4a82cd1, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/ac4a4ebc2d5b48379c26adaa84721500, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/2c2f9f9708a54e12badd33d38b2c2677, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/dbf23d703a7349e4b31a58cc0423175d, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/37eaac5afbed447ca4c8e176a7a568ad, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/48a746ee70c0450e8f1196c82d6ec8b9, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/c53448bd0f554ff8b64dc65c8785b77f, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/d83a9484443546298b44a592d1a0ba97, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/4ae44de7a41c4fafba785b679aa151f4, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/bbb108a1f633472093f816f62ae77c4e, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/6260e97cfe224c17b8913b944009fd8d, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/7233652bcf1f480f8ff33dec83b46f57, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/f28cc6129af8492683331ffd31d4db7c] to archive 2024-12-04T19:16:48,761 DEBUG [StoreCloser-TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-04T19:16:48,762 DEBUG [StoreCloser-TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/79ccd2ff28b54a96842ea98b8af18281 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/79ccd2ff28b54a96842ea98b8af18281 2024-12-04T19:16:48,763 DEBUG [StoreCloser-TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/bd2202b4a0a54318a5b6cd829d6a1118 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/bd2202b4a0a54318a5b6cd829d6a1118 2024-12-04T19:16:48,763 DEBUG [StoreCloser-TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/0127ace727de4eeca25fc4c7b6984a09 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/0127ace727de4eeca25fc4c7b6984a09 2024-12-04T19:16:48,764 DEBUG [StoreCloser-TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/6c24293234724f478ca10a56c33300d4 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/6c24293234724f478ca10a56c33300d4 2024-12-04T19:16:48,765 DEBUG [StoreCloser-TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/01b887b3f0cd49d49bb361c4e67c6fc8 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/01b887b3f0cd49d49bb361c4e67c6fc8 2024-12-04T19:16:48,766 DEBUG [StoreCloser-TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/c06df5860725447fbac6bcef54a9402c to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/c06df5860725447fbac6bcef54a9402c 2024-12-04T19:16:48,767 DEBUG [StoreCloser-TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/7c1857b1f5d544d0a6a56699ea2b9e6a to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/7c1857b1f5d544d0a6a56699ea2b9e6a 2024-12-04T19:16:48,767 DEBUG [StoreCloser-TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/e284e87bc2f643d9a1018db9a18f4560 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/e284e87bc2f643d9a1018db9a18f4560 2024-12-04T19:16:48,768 DEBUG [StoreCloser-TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/5aaea413efe6478e9485ac530987a91f to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/5aaea413efe6478e9485ac530987a91f 2024-12-04T19:16:48,769 DEBUG [StoreCloser-TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/becd0bfa340543349ddc8f2e15c8f77d to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/becd0bfa340543349ddc8f2e15c8f77d 2024-12-04T19:16:48,769 DEBUG [StoreCloser-TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/8f7d3df6a4ae48c8a9e97619f4a82cd1 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/8f7d3df6a4ae48c8a9e97619f4a82cd1 2024-12-04T19:16:48,770 DEBUG [StoreCloser-TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/ac4a4ebc2d5b48379c26adaa84721500 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/ac4a4ebc2d5b48379c26adaa84721500 2024-12-04T19:16:48,770 DEBUG [StoreCloser-TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/2c2f9f9708a54e12badd33d38b2c2677 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/2c2f9f9708a54e12badd33d38b2c2677 2024-12-04T19:16:48,771 DEBUG [StoreCloser-TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/dbf23d703a7349e4b31a58cc0423175d to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/dbf23d703a7349e4b31a58cc0423175d 2024-12-04T19:16:48,771 DEBUG [StoreCloser-TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/37eaac5afbed447ca4c8e176a7a568ad to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/37eaac5afbed447ca4c8e176a7a568ad 2024-12-04T19:16:48,772 DEBUG [StoreCloser-TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/48a746ee70c0450e8f1196c82d6ec8b9 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/48a746ee70c0450e8f1196c82d6ec8b9 2024-12-04T19:16:48,772 DEBUG [StoreCloser-TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/c53448bd0f554ff8b64dc65c8785b77f to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/c53448bd0f554ff8b64dc65c8785b77f 2024-12-04T19:16:48,773 DEBUG [StoreCloser-TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/d83a9484443546298b44a592d1a0ba97 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/d83a9484443546298b44a592d1a0ba97 2024-12-04T19:16:48,774 DEBUG [StoreCloser-TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/4ae44de7a41c4fafba785b679aa151f4 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/4ae44de7a41c4fafba785b679aa151f4 2024-12-04T19:16:48,774 DEBUG [StoreCloser-TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/bbb108a1f633472093f816f62ae77c4e to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/bbb108a1f633472093f816f62ae77c4e 2024-12-04T19:16:48,775 DEBUG [StoreCloser-TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/6260e97cfe224c17b8913b944009fd8d to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/6260e97cfe224c17b8913b944009fd8d 2024-12-04T19:16:48,775 DEBUG [StoreCloser-TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/7233652bcf1f480f8ff33dec83b46f57 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/7233652bcf1f480f8ff33dec83b46f57 2024-12-04T19:16:48,776 DEBUG [StoreCloser-TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/f28cc6129af8492683331ffd31d4db7c to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/f28cc6129af8492683331ffd31d4db7c 2024-12-04T19:16:48,779 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/recovered.edits/783.seqid, newMaxSeqId=783, maxSeqId=4 2024-12-04T19:16:48,779 INFO [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. 2024-12-04T19:16:48,779 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1676): Region close journal for e22a2b2fd059f99265758fd39ea40832: Waiting for close lock at 1733339807015Running coprocessor pre-close hooks at 1733339807015Disabling compacts and flushes for region at 1733339807015Disabling writes for close at 1733339807015Obtaining lock to block concurrent updates at 1733339807015Preparing flush snapshotting stores in e22a2b2fd059f99265758fd39ea40832 at 1733339807015Finished memstore snapshotting TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1733339807016 (+1 ms)Flushing stores of TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832. at 1733339807016Flushing e22a2b2fd059f99265758fd39ea40832/A: creating writer at 1733339807016Flushing mob file e22a2b2fd059f99265758fd39ea40832/A: appending metadata at 1733339807020 (+4 ms)Flushing mob file e22a2b2fd059f99265758fd39ea40832/A: closing flushed file at 1733339807020Flushing e22a2b2fd059f99265758fd39ea40832/A: appending metadata at 1733339807433 (+413 ms)Flushing e22a2b2fd059f99265758fd39ea40832/A: closing flushed file at 1733339807433Flushing e22a2b2fd059f99265758fd39ea40832/B: creating writer at 1733339807849 (+416 ms)Flushing e22a2b2fd059f99265758fd39ea40832/B: appending metadata at 1733339807852 (+3 ms)Flushing e22a2b2fd059f99265758fd39ea40832/B: closing flushed file at 1733339807852Flushing e22a2b2fd059f99265758fd39ea40832/C: creating writer at 1733339808268 (+416 ms)Flushing e22a2b2fd059f99265758fd39ea40832/C: appending metadata at 1733339808272 (+4 ms)Flushing e22a2b2fd059f99265758fd39ea40832/C: closing flushed file at 1733339808272Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7fcf6805: reopening flushed file at 1733339808687 (+415 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7d65812a: reopening flushed file at 1733339808694 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@741fcb2b: reopening flushed file at 1733339808698 (+4 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for e22a2b2fd059f99265758fd39ea40832 in 1687ms, sequenceid=780, compaction requested=true at 1733339808702 (+4 ms)Writing region close event to WAL at 1733339808777 (+75 ms)Running coprocessor post-close hooks at 1733339808779 (+2 ms)Closed at 1733339808779 2024-12-04T19:16:48,781 INFO [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] handler.UnassignRegionHandler(157): Closed e22a2b2fd059f99265758fd39ea40832 2024-12-04T19:16:48,782 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=164 updating hbase:meta row=e22a2b2fd059f99265758fd39ea40832, regionState=CLOSED 2024-12-04T19:16:48,783 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=165, ppid=164, state=RUNNABLE, hasLock=false; CloseRegionProcedure e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 because future has completed 2024-12-04T19:16:48,785 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-12-04T19:16:48,786 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; CloseRegionProcedure e22a2b2fd059f99265758fd39ea40832, server=9ba2734cd39f,33969,1733339629791 in 1.9200 sec 2024-12-04T19:16:48,787 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=164, resume processing ppid=163 2024-12-04T19:16:48,787 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, ppid=163, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e22a2b2fd059f99265758fd39ea40832, UNASSIGN in 1.9250 sec 2024-12-04T19:16:48,789 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-12-04T19:16:48,789 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9280 sec 2024-12-04T19:16:48,791 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733339808790"}]},"ts":"1733339808790"} 2024-12-04T19:16:48,792 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-04T19:16:48,792 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-04T19:16:48,794 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9380 sec 2024-12-04T19:16:49,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-04T19:16:49,013 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-04T19:16:49,014 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-04T19:16:49,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T19:16:49,017 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=166, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T19:16:49,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-04T19:16:49,019 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=166, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T19:16:49,023 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832 2024-12-04T19:16:49,026 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A, FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B, FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C, FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/recovered.edits] 2024-12-04T19:16:49,029 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/2593637b2b9849e8bb453b0391820449 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/2593637b2b9849e8bb453b0391820449 2024-12-04T19:16:49,031 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/58a47ba3d7ac4a15bfd9f14ff6a12fa9 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/58a47ba3d7ac4a15bfd9f14ff6a12fa9 2024-12-04T19:16:49,032 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/b066c092d0004dd59b5a12c4e8f4f702 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/A/b066c092d0004dd59b5a12c4e8f4f702 2024-12-04T19:16:49,035 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/a5eaed1ac6d8445f9e22b8589cdc83fe to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/a5eaed1ac6d8445f9e22b8589cdc83fe 2024-12-04T19:16:49,037 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/b19ba86cdff54b8bb05ff38cd8bf6cc9 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/b19ba86cdff54b8bb05ff38cd8bf6cc9 2024-12-04T19:16:49,039 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/feb36dca1753400ca82ce378f4bae4ff to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/B/feb36dca1753400ca82ce378f4bae4ff 2024-12-04T19:16:49,043 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/2254c3e1309e4d2fa23a96583d86d2cb to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/2254c3e1309e4d2fa23a96583d86d2cb 2024-12-04T19:16:49,045 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/3420eaf30ea74b9c8dc6cc92442c6aab to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/3420eaf30ea74b9c8dc6cc92442c6aab 2024-12-04T19:16:49,047 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/a1cdeb3e281242d38919968262bc67e1 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/C/a1cdeb3e281242d38919968262bc67e1 2024-12-04T19:16:49,051 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/recovered.edits/783.seqid to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832/recovered.edits/783.seqid 2024-12-04T19:16:49,052 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/default/TestAcidGuarantees/e22a2b2fd059f99265758fd39ea40832 2024-12-04T19:16:49,052 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-04T19:16:49,053 DEBUG [PEWorker-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-04T19:16:49,054 DEBUG [PEWorker-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-04T19:16:49,060 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204155d65f8f8f94b92915492ca07a9ce52_e22a2b2fd059f99265758fd39ea40832 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204155d65f8f8f94b92915492ca07a9ce52_e22a2b2fd059f99265758fd39ea40832 2024-12-04T19:16:49,062 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412043256a685911b4829953bf407724e052e_e22a2b2fd059f99265758fd39ea40832 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412043256a685911b4829953bf407724e052e_e22a2b2fd059f99265758fd39ea40832 2024-12-04T19:16:49,064 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120447347198501c4dc1b69339cfb7a94899_e22a2b2fd059f99265758fd39ea40832 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120447347198501c4dc1b69339cfb7a94899_e22a2b2fd059f99265758fd39ea40832 2024-12-04T19:16:49,065 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120456c8e403981f4a379eb5c686114b85b2_e22a2b2fd059f99265758fd39ea40832 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120456c8e403981f4a379eb5c686114b85b2_e22a2b2fd059f99265758fd39ea40832 2024-12-04T19:16:49,067 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412045d9a1797c8154f1a880ea1c036af953d_e22a2b2fd059f99265758fd39ea40832 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412045d9a1797c8154f1a880ea1c036af953d_e22a2b2fd059f99265758fd39ea40832 2024-12-04T19:16:49,069 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412045f1b538192ce47d3aa56fef69c439f11_e22a2b2fd059f99265758fd39ea40832 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412045f1b538192ce47d3aa56fef69c439f11_e22a2b2fd059f99265758fd39ea40832 2024-12-04T19:16:49,071 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412046dd229654b714e59bd1b3b3d132b21f4_e22a2b2fd059f99265758fd39ea40832 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412046dd229654b714e59bd1b3b3d132b21f4_e22a2b2fd059f99265758fd39ea40832 2024-12-04T19:16:49,073 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412047333ff4a42a24cd4930a0813d661233f_e22a2b2fd059f99265758fd39ea40832 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412047333ff4a42a24cd4930a0813d661233f_e22a2b2fd059f99265758fd39ea40832 2024-12-04T19:16:49,074 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412047cf1f19b4898407d91d29a31b65b1535_e22a2b2fd059f99265758fd39ea40832 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412047cf1f19b4898407d91d29a31b65b1535_e22a2b2fd059f99265758fd39ea40832 2024-12-04T19:16:49,076 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120483703369dc0a4938a00cadc041f49dc8_e22a2b2fd059f99265758fd39ea40832 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120483703369dc0a4938a00cadc041f49dc8_e22a2b2fd059f99265758fd39ea40832 2024-12-04T19:16:49,077 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412048745c6591dbc4110b951d901ddf736b2_e22a2b2fd059f99265758fd39ea40832 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412048745c6591dbc4110b951d901ddf736b2_e22a2b2fd059f99265758fd39ea40832 2024-12-04T19:16:49,079 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412048fae1f7f6c09490ba3eb1f4273545f21_e22a2b2fd059f99265758fd39ea40832 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412048fae1f7f6c09490ba3eb1f4273545f21_e22a2b2fd059f99265758fd39ea40832 2024-12-04T19:16:49,080 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204998316d32a504546a149751f5cfa269a_e22a2b2fd059f99265758fd39ea40832 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204998316d32a504546a149751f5cfa269a_e22a2b2fd059f99265758fd39ea40832 2024-12-04T19:16:49,081 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412049d9f461835be48c393b66493cadaf452_e22a2b2fd059f99265758fd39ea40832 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412049d9f461835be48c393b66493cadaf452_e22a2b2fd059f99265758fd39ea40832 2024-12-04T19:16:49,083 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412049e8c2dfeab6c480a8f0fbe9cc2748dc4_e22a2b2fd059f99265758fd39ea40832 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412049e8c2dfeab6c480a8f0fbe9cc2748dc4_e22a2b2fd059f99265758fd39ea40832 2024-12-04T19:16:49,084 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204c2b0560abcc244939cecdada755ee168_e22a2b2fd059f99265758fd39ea40832 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204c2b0560abcc244939cecdada755ee168_e22a2b2fd059f99265758fd39ea40832 2024-12-04T19:16:49,085 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204d092d7597a2449f4bf8a419c8ea9e67a_e22a2b2fd059f99265758fd39ea40832 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204d092d7597a2449f4bf8a419c8ea9e67a_e22a2b2fd059f99265758fd39ea40832 2024-12-04T19:16:49,087 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204d88f62edc2114bcc9f355174e75dfd2c_e22a2b2fd059f99265758fd39ea40832 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204d88f62edc2114bcc9f355174e75dfd2c_e22a2b2fd059f99265758fd39ea40832 2024-12-04T19:16:49,088 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204e2374b0a89904335995a74aa874b4fce_e22a2b2fd059f99265758fd39ea40832 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204e2374b0a89904335995a74aa874b4fce_e22a2b2fd059f99265758fd39ea40832 2024-12-04T19:16:49,089 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204f9b244e57d4b49e4a7a9e2eb1c292689_e22a2b2fd059f99265758fd39ea40832 to hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204f9b244e57d4b49e4a7a9e2eb1c292689_e22a2b2fd059f99265758fd39ea40832 2024-12-04T19:16:49,089 DEBUG [PEWorker-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-04T19:16:49,092 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=166, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T19:16:49,094 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-04T19:16:49,096 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-04T19:16:49,097 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=166, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T19:16:49,097 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-04T19:16:49,097 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733339809097"}]},"ts":"9223372036854775807"} 2024-12-04T19:16:49,099 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-04T19:16:49,099 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => e22a2b2fd059f99265758fd39ea40832, NAME => 'TestAcidGuarantees,,1733339775956.e22a2b2fd059f99265758fd39ea40832.', STARTKEY => '', ENDKEY => ''}] 2024-12-04T19:16:49,099 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-04T19:16:49,099 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733339809099"}]},"ts":"9223372036854775807"} 2024-12-04T19:16:49,101 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-04T19:16:49,102 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=166, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T19:16:49,103 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 88 msec 2024-12-04T19:16:49,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46667 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-04T19:16:49,123 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-04T19:16:49,123 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-04T19:16:49,135 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=247 (was 250), OpenFileDescriptor=445 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=392 (was 471), ProcessCount=11 (was 11), AvailableMemoryMB=14971 (was 15133) 2024-12-04T19:16:49,135 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-04T19:16:49,136 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-04T19:16:49,136 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T19:16:49,136 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:16:49,136 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:16:49,136 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T19:16:49,136 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-04T19:16:49,136 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1519452659, stopped=false 2024-12-04T19:16:49,137 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=9ba2734cd39f,46667,1733339628877 2024-12-04T19:16:49,189 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-04T19:16:49,210 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46667-0x101a1c3bdc90000, quorum=127.0.0.1:60754, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-04T19:16:49,210 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33969-0x101a1c3bdc90001, quorum=127.0.0.1:60754, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-04T19:16:49,210 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46667-0x101a1c3bdc90000, quorum=127.0.0.1:60754, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T19:16:49,210 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33969-0x101a1c3bdc90001, quorum=127.0.0.1:60754, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T19:16:49,210 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-04T19:16:49,211 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-04T19:16:49,211 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T19:16:49,211 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:16:49,211 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '9ba2734cd39f,33969,1733339629791' ***** 2024-12-04T19:16:49,211 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-04T19:16:49,212 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:33969-0x101a1c3bdc90001, quorum=127.0.0.1:60754, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T19:16:49,212 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:46667-0x101a1c3bdc90000, quorum=127.0.0.1:60754, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T19:16:49,212 INFO [RS:0;9ba2734cd39f:33969 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-04T19:16:49,212 INFO [RS:0;9ba2734cd39f:33969 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-04T19:16:49,212 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-04T19:16:49,212 INFO [RS:0;9ba2734cd39f:33969 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-04T19:16:49,212 INFO [RS:0;9ba2734cd39f:33969 {}] regionserver.HRegionServer(959): stopping server 9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:49,212 INFO [RS:0;9ba2734cd39f:33969 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-04T19:16:49,213 INFO [RS:0;9ba2734cd39f:33969 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;9ba2734cd39f:33969. 2024-12-04T19:16:49,213 DEBUG [RS:0;9ba2734cd39f:33969 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T19:16:49,213 DEBUG [RS:0;9ba2734cd39f:33969 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:16:49,213 INFO [RS:0;9ba2734cd39f:33969 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-04T19:16:49,213 INFO [RS:0;9ba2734cd39f:33969 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-04T19:16:49,213 INFO [RS:0;9ba2734cd39f:33969 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-04T19:16:49,213 INFO [RS:0;9ba2734cd39f:33969 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-04T19:16:49,213 INFO [RS:0;9ba2734cd39f:33969 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-04T19:16:49,214 DEBUG [RS:0;9ba2734cd39f:33969 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-12-04T19:16:49,214 DEBUG [RS:0;9ba2734cd39f:33969 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-04T19:16:49,214 DEBUG [RS_CLOSE_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-04T19:16:49,214 INFO [RS_CLOSE_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-04T19:16:49,214 DEBUG [RS_CLOSE_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-04T19:16:49,214 DEBUG [RS_CLOSE_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-04T19:16:49,214 DEBUG [RS_CLOSE_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-04T19:16:49,214 INFO [RS_CLOSE_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-12-04T19:16:49,233 DEBUG [RS_CLOSE_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/hbase/meta/1588230740/.tmp/info/0a22b971e5a748be93e4bcc820e4de7d is 95, key is TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5./info:/1733339666950/DeleteFamily/seqid=0 2024-12-04T19:16:49,235 INFO [regionserver/9ba2734cd39f:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-04T19:16:49,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742466_1642 (size=6514) 2024-12-04T19:16:49,360 INFO [regionserver/9ba2734cd39f:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-04T19:16:49,360 INFO [regionserver/9ba2734cd39f:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-04T19:16:49,414 DEBUG [RS:0;9ba2734cd39f:33969 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-04T19:16:49,614 DEBUG [RS:0;9ba2734cd39f:33969 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-04T19:16:49,637 INFO [RS_CLOSE_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/hbase/meta/1588230740/.tmp/info/0a22b971e5a748be93e4bcc820e4de7d 2024-12-04T19:16:49,653 DEBUG [RS_CLOSE_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/hbase/meta/1588230740/.tmp/ns/8f357f55d61e43ecb9080438e3742420 is 93, key is TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5./ns:/1733339666910/DeleteFamily/seqid=0 2024-12-04T19:16:49,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742467_1643 (size=5906) 2024-12-04T19:16:49,815 DEBUG [RS:0;9ba2734cd39f:33969 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-04T19:16:50,015 DEBUG [RS:0;9ba2734cd39f:33969 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-04T19:16:50,057 INFO [RS_CLOSE_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/hbase/meta/1588230740/.tmp/ns/8f357f55d61e43ecb9080438e3742420 2024-12-04T19:16:50,073 DEBUG [RS_CLOSE_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/hbase/meta/1588230740/.tmp/rep_barrier/70d4f6d4be144c809317b6bca3473480 is 102, key is TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5./rep_barrier:/1733339666910/DeleteFamily/seqid=0 2024-12-04T19:16:50,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742468_1644 (size=6025) 2024-12-04T19:16:50,215 DEBUG [RS:0;9ba2734cd39f:33969 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-04T19:16:50,416 DEBUG [RS:0;9ba2734cd39f:33969 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-04T19:16:50,479 INFO [RS_CLOSE_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/hbase/meta/1588230740/.tmp/rep_barrier/70d4f6d4be144c809317b6bca3473480 2024-12-04T19:16:50,536 DEBUG [RS_CLOSE_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/hbase/meta/1588230740/.tmp/table/04c7dbf4155c4e909e5989baaa084c5d is 96, key is TestAcidGuarantees,,1733339634085.97532d2585300ca10ffc4f6d5dda82c5./table:/1733339666910/DeleteFamily/seqid=0 2024-12-04T19:16:50,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742469_1645 (size=5936) 2024-12-04T19:16:50,616 DEBUG [RS:0;9ba2734cd39f:33969 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-04T19:16:50,816 DEBUG [RS:0;9ba2734cd39f:33969 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-04T19:16:50,941 INFO [RS_CLOSE_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/hbase/meta/1588230740/.tmp/table/04c7dbf4155c4e909e5989baaa084c5d 2024-12-04T19:16:50,950 DEBUG [RS_CLOSE_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/hbase/meta/1588230740/.tmp/info/0a22b971e5a748be93e4bcc820e4de7d as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/hbase/meta/1588230740/info/0a22b971e5a748be93e4bcc820e4de7d 2024-12-04T19:16:50,954 INFO [RS_CLOSE_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/hbase/meta/1588230740/info/0a22b971e5a748be93e4bcc820e4de7d, entries=12, sequenceid=90, filesize=6.4 K 2024-12-04T19:16:50,955 DEBUG [RS_CLOSE_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/hbase/meta/1588230740/.tmp/ns/8f357f55d61e43ecb9080438e3742420 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/hbase/meta/1588230740/ns/8f357f55d61e43ecb9080438e3742420 2024-12-04T19:16:50,958 INFO [RS_CLOSE_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/hbase/meta/1588230740/ns/8f357f55d61e43ecb9080438e3742420, entries=8, sequenceid=90, filesize=5.8 K 2024-12-04T19:16:50,959 DEBUG [RS_CLOSE_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/hbase/meta/1588230740/.tmp/rep_barrier/70d4f6d4be144c809317b6bca3473480 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/hbase/meta/1588230740/rep_barrier/70d4f6d4be144c809317b6bca3473480 2024-12-04T19:16:50,963 INFO [RS_CLOSE_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/hbase/meta/1588230740/rep_barrier/70d4f6d4be144c809317b6bca3473480, entries=6, sequenceid=90, filesize=5.9 K 2024-12-04T19:16:50,963 DEBUG [RS_CLOSE_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/hbase/meta/1588230740/.tmp/table/04c7dbf4155c4e909e5989baaa084c5d as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/hbase/meta/1588230740/table/04c7dbf4155c4e909e5989baaa084c5d 2024-12-04T19:16:50,967 INFO [RS_CLOSE_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/hbase/meta/1588230740/table/04c7dbf4155c4e909e5989baaa084c5d, entries=7, sequenceid=90, filesize=5.8 K 2024-12-04T19:16:50,968 INFO [RS_CLOSE_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1754ms, sequenceid=90, compaction requested=false 2024-12-04T19:16:50,973 DEBUG [RS_CLOSE_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-12-04T19:16:50,973 DEBUG [RS_CLOSE_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-04T19:16:50,973 INFO [RS_CLOSE_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-04T19:16:50,973 DEBUG [RS_CLOSE_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733339809213Running coprocessor pre-close hooks at 1733339809214 (+1 ms)Disabling compacts and flushes for region at 1733339809214Disabling writes for close at 1733339809214Obtaining lock to block concurrent updates at 1733339809214Preparing flush snapshotting stores in 1588230740 at 1733339809214Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1733339809214Flushing stores of hbase:meta,,1.1588230740 at 1733339809215 (+1 ms)Flushing 1588230740/info: creating writer at 1733339809215Flushing 1588230740/info: appending metadata at 1733339809233 (+18 ms)Flushing 1588230740/info: closing flushed file at 1733339809233Flushing 1588230740/ns: creating writer at 1733339809640 (+407 ms)Flushing 1588230740/ns: appending metadata at 1733339809653 (+13 ms)Flushing 1588230740/ns: closing flushed file at 1733339809653Flushing 1588230740/rep_barrier: creating writer at 1733339810060 (+407 ms)Flushing 1588230740/rep_barrier: appending metadata at 1733339810073 (+13 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1733339810073Flushing 1588230740/table: creating writer at 1733339810523 (+450 ms)Flushing 1588230740/table: appending metadata at 1733339810536 (+13 ms)Flushing 1588230740/table: closing flushed file at 1733339810536Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3b45663f: reopening flushed file at 1733339810950 (+414 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3634c312: reopening flushed file at 1733339810954 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4e537b2: reopening flushed file at 1733339810958 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@42b01cd4: reopening flushed file at 1733339810963 (+5 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1754ms, sequenceid=90, compaction requested=false at 1733339810968 (+5 ms)Writing region close event to WAL at 1733339810970 (+2 ms)Running coprocessor post-close hooks at 1733339810973 (+3 ms)Closed at 1733339810973 2024-12-04T19:16:50,973 DEBUG [RS_CLOSE_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-04T19:16:51,017 INFO [RS:0;9ba2734cd39f:33969 {}] regionserver.HRegionServer(976): stopping server 9ba2734cd39f,33969,1733339629791; all regions closed. 2024-12-04T19:16:51,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741834_1010 (size=25344) 2024-12-04T19:16:51,026 DEBUG [RS:0;9ba2734cd39f:33969 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/oldWALs 2024-12-04T19:16:51,026 INFO [RS:0;9ba2734cd39f:33969 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 9ba2734cd39f%2C33969%2C1733339629791.meta:.meta(num 1733339633576) 2024-12-04T19:16:51,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741832_1008 (size=22630296) 2024-12-04T19:16:51,032 DEBUG [RS:0;9ba2734cd39f:33969 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/oldWALs 2024-12-04T19:16:51,032 INFO [RS:0;9ba2734cd39f:33969 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 9ba2734cd39f%2C33969%2C1733339629791:(num 1733339632500) 2024-12-04T19:16:51,032 DEBUG [RS:0;9ba2734cd39f:33969 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:16:51,032 INFO [RS:0;9ba2734cd39f:33969 {}] regionserver.LeaseManager(133): Closed leases 2024-12-04T19:16:51,032 INFO [RS:0;9ba2734cd39f:33969 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-04T19:16:51,033 INFO [RS:0;9ba2734cd39f:33969 {}] hbase.ChoreService(370): Chore service for: regionserver/9ba2734cd39f:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-04T19:16:51,033 INFO [RS:0;9ba2734cd39f:33969 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-04T19:16:51,033 INFO [regionserver/9ba2734cd39f:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-04T19:16:51,035 INFO [RS:0;9ba2734cd39f:33969 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33969 2024-12-04T19:16:51,084 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33969-0x101a1c3bdc90001, quorum=127.0.0.1:60754, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/9ba2734cd39f,33969,1733339629791 2024-12-04T19:16:51,084 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46667-0x101a1c3bdc90000, quorum=127.0.0.1:60754, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-04T19:16:51,084 INFO [RS:0;9ba2734cd39f:33969 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-04T19:16:51,085 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [9ba2734cd39f,33969,1733339629791] 2024-12-04T19:16:51,105 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/9ba2734cd39f,33969,1733339629791 already deleted, retry=false 2024-12-04T19:16:51,105 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 9ba2734cd39f,33969,1733339629791 expired; onlineServers=0 2024-12-04T19:16:51,106 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '9ba2734cd39f,46667,1733339628877' ***** 2024-12-04T19:16:51,106 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-04T19:16:51,106 INFO [M:0;9ba2734cd39f:46667 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-04T19:16:51,106 INFO [M:0;9ba2734cd39f:46667 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-04T19:16:51,107 DEBUG [M:0;9ba2734cd39f:46667 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-04T19:16:51,107 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-04T19:16:51,107 DEBUG [M:0;9ba2734cd39f:46667 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-04T19:16:51,107 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster-HFileCleaner.small.0-1733339632228 {}] cleaner.HFileCleaner(306): Exit Thread[master/9ba2734cd39f:0:becomeActiveMaster-HFileCleaner.small.0-1733339632228,5,FailOnTimeoutGroup] 2024-12-04T19:16:51,107 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster-HFileCleaner.large.0-1733339632226 {}] cleaner.HFileCleaner(306): Exit Thread[master/9ba2734cd39f:0:becomeActiveMaster-HFileCleaner.large.0-1733339632226,5,FailOnTimeoutGroup] 2024-12-04T19:16:51,107 INFO [M:0;9ba2734cd39f:46667 {}] hbase.ChoreService(370): Chore service for: master/9ba2734cd39f:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-04T19:16:51,108 INFO [M:0;9ba2734cd39f:46667 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-04T19:16:51,108 DEBUG [M:0;9ba2734cd39f:46667 {}] master.HMaster(1795): Stopping service threads 2024-12-04T19:16:51,108 INFO [M:0;9ba2734cd39f:46667 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-04T19:16:51,108 INFO [M:0;9ba2734cd39f:46667 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-04T19:16:51,108 ERROR [M:0;9ba2734cd39f:46667 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-12-04T19:16:51,109 INFO [M:0;9ba2734cd39f:46667 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-04T19:16:51,109 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-04T19:16:51,115 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46667-0x101a1c3bdc90000, quorum=127.0.0.1:60754, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-04T19:16:51,115 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46667-0x101a1c3bdc90000, quorum=127.0.0.1:60754, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T19:16:51,115 DEBUG [M:0;9ba2734cd39f:46667 {}] zookeeper.ZKUtil(347): master:46667-0x101a1c3bdc90000, quorum=127.0.0.1:60754, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-04T19:16:51,116 WARN [M:0;9ba2734cd39f:46667 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-04T19:16:51,116 INFO [M:0;9ba2734cd39f:46667 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/.lastflushedseqids 2024-12-04T19:16:51,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742470_1646 (size=72) 2024-12-04T19:16:51,196 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33969-0x101a1c3bdc90001, quorum=127.0.0.1:60754, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T19:16:51,196 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33969-0x101a1c3bdc90001, quorum=127.0.0.1:60754, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T19:16:51,197 INFO [RS:0;9ba2734cd39f:33969 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-04T19:16:51,197 INFO [RS:0;9ba2734cd39f:33969 {}] regionserver.HRegionServer(1031): Exiting; stopping=9ba2734cd39f,33969,1733339629791; zookeeper connection closed. 2024-12-04T19:16:51,198 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@6b790911 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@6b790911 2024-12-04T19:16:51,199 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-12-04T19:16:51,529 INFO [M:0;9ba2734cd39f:46667 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-04T19:16:51,529 INFO [M:0;9ba2734cd39f:46667 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-04T19:16:51,529 DEBUG [M:0;9ba2734cd39f:46667 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-04T19:16:51,529 INFO [M:0;9ba2734cd39f:46667 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T19:16:51,529 DEBUG [M:0;9ba2734cd39f:46667 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T19:16:51,529 DEBUG [M:0;9ba2734cd39f:46667 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-04T19:16:51,530 DEBUG [M:0;9ba2734cd39f:46667 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T19:16:51,530 INFO [M:0;9ba2734cd39f:46667 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=763.21 KB heapSize=934.09 KB 2024-12-04T19:16:51,551 DEBUG [M:0;9ba2734cd39f:46667 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/00cba14a904e4205af59f93419f02478 is 82, key is hbase:meta,,1/info:regioninfo/1733339633655/Put/seqid=0 2024-12-04T19:16:51,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742471_1647 (size=5672) 2024-12-04T19:16:51,955 INFO [M:0;9ba2734cd39f:46667 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2108 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/00cba14a904e4205af59f93419f02478 2024-12-04T19:16:51,984 DEBUG [M:0;9ba2734cd39f:46667 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/15a4a442b62f4a1e9809c92ae8d930c2 is 2480, key is \x00\x00\x00\x00\x00\x00\x00\x8F/proc:d/1733339777160/Put/seqid=0 2024-12-04T19:16:51,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742472_1648 (size=44661) 2024-12-04T19:16:52,387 INFO [M:0;9ba2734cd39f:46667 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=762.65 KB at sequenceid=2108 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/15a4a442b62f4a1e9809c92ae8d930c2 2024-12-04T19:16:52,392 INFO [M:0;9ba2734cd39f:46667 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 15a4a442b62f4a1e9809c92ae8d930c2 2024-12-04T19:16:52,409 DEBUG [M:0;9ba2734cd39f:46667 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/6bd47d07f3314db78186a2cecd7b3ad0 is 69, key is 9ba2734cd39f,33969,1733339629791/rs:state/1733339632270/Put/seqid=0 2024-12-04T19:16:52,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073742473_1649 (size=5156) 2024-12-04T19:16:52,813 INFO [M:0;9ba2734cd39f:46667 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2108 (bloomFilter=true), to=hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/6bd47d07f3314db78186a2cecd7b3ad0 2024-12-04T19:16:52,818 DEBUG [M:0;9ba2734cd39f:46667 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/00cba14a904e4205af59f93419f02478 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/00cba14a904e4205af59f93419f02478 2024-12-04T19:16:52,823 INFO [M:0;9ba2734cd39f:46667 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/00cba14a904e4205af59f93419f02478, entries=8, sequenceid=2108, filesize=5.5 K 2024-12-04T19:16:52,824 DEBUG [M:0;9ba2734cd39f:46667 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/15a4a442b62f4a1e9809c92ae8d930c2 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/15a4a442b62f4a1e9809c92ae8d930c2 2024-12-04T19:16:52,828 INFO [M:0;9ba2734cd39f:46667 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 15a4a442b62f4a1e9809c92ae8d930c2 2024-12-04T19:16:52,828 INFO [M:0;9ba2734cd39f:46667 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/15a4a442b62f4a1e9809c92ae8d930c2, entries=166, sequenceid=2108, filesize=43.6 K 2024-12-04T19:16:52,829 DEBUG [M:0;9ba2734cd39f:46667 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/6bd47d07f3314db78186a2cecd7b3ad0 as hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/6bd47d07f3314db78186a2cecd7b3ad0 2024-12-04T19:16:52,832 INFO [M:0;9ba2734cd39f:46667 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44307/user/jenkins/test-data/16e85e44-6d87-c8bc-172b-f2aa8e56962d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/6bd47d07f3314db78186a2cecd7b3ad0, entries=1, sequenceid=2108, filesize=5.0 K 2024-12-04T19:16:52,833 INFO [M:0;9ba2734cd39f:46667 {}] regionserver.HRegion(3140): Finished flush of dataSize ~763.21 KB/781526, heapSize ~933.79 KB/956200, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1303ms, sequenceid=2108, compaction requested=false 2024-12-04T19:16:52,834 INFO [M:0;9ba2734cd39f:46667 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T19:16:52,834 DEBUG [M:0;9ba2734cd39f:46667 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733339811529Disabling compacts and flushes for region at 1733339811529Disabling writes for close at 1733339811530 (+1 ms)Obtaining lock to block concurrent updates at 1733339811530Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733339811530Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=781526, getHeapSize=956440, getOffHeapSize=0, getCellsCount=2299 at 1733339811530Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733339811531 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733339811532 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733339811551 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733339811551Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733339811962 (+411 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733339811983 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733339811983Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733339812392 (+409 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733339812409 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733339812409Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@663bbc3: reopening flushed file at 1733339812817 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7d74902a: reopening flushed file at 1733339812824 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@a38a3e8: reopening flushed file at 1733339812828 (+4 ms)Finished flush of dataSize ~763.21 KB/781526, heapSize ~933.79 KB/956200, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1303ms, sequenceid=2108, compaction requested=false at 1733339812833 (+5 ms)Writing region close event to WAL at 1733339812834 (+1 ms)Closed at 1733339812834 2024-12-04T19:16:52,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34819 is added to blk_1073741830_1006 (size=919729) 2024-12-04T19:16:52,837 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-04T19:16:52,838 INFO [M:0;9ba2734cd39f:46667 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-04T19:16:52,838 INFO [M:0;9ba2734cd39f:46667 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:46667 2024-12-04T19:16:52,838 INFO [M:0;9ba2734cd39f:46667 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-04T19:16:52,984 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46667-0x101a1c3bdc90000, quorum=127.0.0.1:60754, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T19:16:52,985 INFO [M:0;9ba2734cd39f:46667 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-04T19:16:52,985 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46667-0x101a1c3bdc90000, quorum=127.0.0.1:60754, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T19:16:52,995 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@42a79998{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T19:16:52,998 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4975be1c{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T19:16:52,998 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T19:16:52,999 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@74fde7a3{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-04T19:16:52,999 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3de8243e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1db4ba39-9940-6e0b-9624-ffcf7fe27afc/hadoop.log.dir/,STOPPED} 2024-12-04T19:16:53,001 WARN [BP-523244764-172.17.0.2-1733339625272 heartbeating to localhost/127.0.0.1:44307 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-04T19:16:53,001 WARN [BP-523244764-172.17.0.2-1733339625272 heartbeating to localhost/127.0.0.1:44307 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-523244764-172.17.0.2-1733339625272 (Datanode Uuid e1a5776f-89d1-4b59-8da9-1063d7e61e79) service to localhost/127.0.0.1:44307 2024-12-04T19:16:53,001 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-04T19:16:53,001 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-04T19:16:53,004 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1db4ba39-9940-6e0b-9624-ffcf7fe27afc/cluster_45aec480-86fd-40c3-b6dd-2ba84309c8cd/data/data1/current/BP-523244764-172.17.0.2-1733339625272 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T19:16:53,004 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1db4ba39-9940-6e0b-9624-ffcf7fe27afc/cluster_45aec480-86fd-40c3-b6dd-2ba84309c8cd/data/data2/current/BP-523244764-172.17.0.2-1733339625272 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T19:16:53,005 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-04T19:16:53,013 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@322e478f{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-04T19:16:53,014 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4b008a6a{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T19:16:53,014 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T19:16:53,014 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@d942d39{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-04T19:16:53,014 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@71c9bba7{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1db4ba39-9940-6e0b-9624-ffcf7fe27afc/hadoop.log.dir/,STOPPED} 2024-12-04T19:16:53,027 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-04T19:16:53,140 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down